From 631d3f59610ee0ea7733b07de3121f17a8aa220d Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 11 Apr 2025 17:00:27 -0700 Subject: [PATCH 01/78] Add scripts to configure sccache-dist and use the build cluster in CI --- .../cuda11.8-conda/devcontainer.json | 4 +- .devcontainer/cuda11.8-pip/devcontainer.json | 4 +- .../cuda12.8-conda/devcontainer.json | 4 +- .devcontainer/cuda12.8-pip/devcontainer.json | 4 +- .devcontainer/rapids.Dockerfile | 16 ++- .../actions/build-and-test-feature/action.yml | 2 + .github/workflows/build-all-rapids-repos.yml | 74 ++++++++-- .github/workflows/build-and-test-feature.yml | 1 + .../devcontainer-feature.json | 2 +- .../bin/generate-scripts.sh | 14 +- .../bin/get-num-archs-jobs-and-load.sh | 62 +-------- .../bin/tmpl/all.cpp.build.tmpl.sh | 31 +++++ .../bin/tmpl/all.python.build.tmpl.sh | 31 +++++ .../bin/tmpl/cpp.configure.tmpl.sh | 1 + .../bin/tmpl/python.build.wheel.tmpl.sh | 17 +++ .../bin/tmpl/python.install.tmpl.sh | 17 +++ features/src/utils/devcontainer-feature.json | 2 +- features/src/utils/install.sh | 7 + .../devcontainer/bin/creds/s3/propagate.sh | 6 +- .../opt/devcontainer/bin/creds/s3/test.sh | 17 +-- .../opt/devcontainer/bin/github/cli/init.sh | 2 +- .../devcontainer/bin/post-attach-command.sh | 26 ++++ .../bin/sccache/dist/configure.sh | 131 ++++++++++++++++++ .../opt/devcontainer/bin/sccache/dist/init.sh | 74 ++++++++++ .../devcontainer/bin/sccache/dist/status.sh | 92 ++++++++++++ .../opt/devcontainer/bin/sccache/install.sh | 111 +++++++++++++++ .../opt/devcontainer/bin/sccache/start.sh | 73 ++++++++++ .../opt/devcontainer/bin/sccache/stop.sh | 63 +++++++++ features/test/utils/scenarios.json | 9 +- features/test/utils/ubuntu18.04.sh | 64 ++++++++- features/test/utils/ubuntu20.04.sh | 64 ++++++++- features/test/utils/ubuntu22.04.sh | 64 ++++++++- 32 files changed, 974 insertions(+), 115 deletions(-) create mode 100755 features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.cpp.build.tmpl.sh create mode 100755 features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.python.build.tmpl.sh create mode 100755 features/src/utils/opt/devcontainer/bin/sccache/dist/configure.sh create mode 100755 features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh create mode 100755 features/src/utils/opt/devcontainer/bin/sccache/dist/status.sh create mode 100755 features/src/utils/opt/devcontainer/bin/sccache/install.sh create mode 100755 features/src/utils/opt/devcontainer/bin/sccache/start.sh create mode 100755 features/src/utils/opt/devcontainer/bin/sccache/stop.sh diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json index c4cd892dd..1b16cc29e 100644 --- a/.devcontainer/cuda11.8-conda/devcontainer.json +++ b/.devcontainer/cuda11.8-conda/devcontainer.json @@ -11,7 +11,9 @@ "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.06-cuda11.8-conda" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.06-cuda11.8-conda", + "--ulimit", + "nofile=1048576:1048576" ], "hostRequirements": {"gpu": "optional"}, "features": { diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index 6616d5ad6..e0b20ddf5 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -11,7 +11,9 @@ "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.06-cuda11.8-pip" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.06-cuda11.8-pip", + "--ulimit", + "nofile=1048576:1048576" ], "hostRequirements": {"gpu": "optional"}, "features": { diff --git a/.devcontainer/cuda12.8-conda/devcontainer.json b/.devcontainer/cuda12.8-conda/devcontainer.json index 077115862..411361ebb 100644 --- a/.devcontainer/cuda12.8-conda/devcontainer.json +++ b/.devcontainer/cuda12.8-conda/devcontainer.json @@ -11,7 +11,9 @@ "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.06-cuda12.8-conda" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.06-cuda12.8-conda", + "--ulimit", + "nofile=1048576:1048576" ], "hostRequirements": {"gpu": "optional"}, "features": { diff --git a/.devcontainer/cuda12.8-pip/devcontainer.json b/.devcontainer/cuda12.8-pip/devcontainer.json index 09375b2b7..257af8a5c 100644 --- a/.devcontainer/cuda12.8-pip/devcontainer.json +++ b/.devcontainer/cuda12.8-pip/devcontainer.json @@ -11,7 +11,9 @@ "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.06-cuda12.8-pip" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.06-cuda12.8-pip", + "--ulimit", + "nofile=1048576:1048576" ], "hostRequirements": {"gpu": "optional"}, "features": { diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile index dde792825..ae5a8b8ac 100644 --- a/.devcontainer/rapids.Dockerfile +++ b/.devcontainer/rapids.Dockerfile @@ -36,6 +36,8 @@ ENV DEFAULT_CONDA_ENV=rapids FROM ${PYTHON_PACKAGE_MANAGER}-base +ARG TARGETARCH + ARG CUDA ENV CUDAARCHS="RAPIDS" ENV CUDA_VERSION="${CUDA_VERSION:-${CUDA}}" @@ -49,9 +51,21 @@ ENV PYTHONDONTWRITEBYTECODE="1" ENV SCCACHE_REGION="us-east-2" ENV SCCACHE_BUCKET="rapids-sccache-devs" -ENV SCCACHE_IDLE_TIMEOUT=900 +ENV SCCACHE_DIST_CONNECT_TIMEOUT=30 +ENV SCCACHE_DIST_REQUEST_TIMEOUT=1800 +ENV SCCACHE_DIST_SCHEDULER_URL="https://${TARGETARCH}.linux.sccache.gha-runners.nvidia.com" +ENV SCCACHE_IDLE_TIMEOUT=1800 ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs" ENV HISTFILE="/home/coder/.cache/._bash_history" ENV LIBCUDF_KERNEL_CACHE_PATH="/home/coder/cudf/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest/jitify_cache" + +# Prevent the sccache server from shutting down +ENV SCCACHE_IDLE_TIMEOUT=0 +ENV SCCACHE_SERVER_LOG="sccache=info" +ENV SCCACHE_S3_KEY_PREFIX=rapids-test-sccache-dist + +# Build as much in parallel as possible +ENV INFER_NUM_DEVICE_ARCHITECTURES=1 +ENV MAX_DEVICE_OBJ_TO_COMPILE_IN_PARALLEL=20 diff --git a/.github/actions/build-and-test-feature/action.yml b/.github/actions/build-and-test-feature/action.yml index 70d11f739..6f3e73ce6 100644 --- a/.github/actions/build-and-test-feature/action.yml +++ b/.github/actions/build-and-test-feature/action.yml @@ -8,6 +8,7 @@ inputs: aws_role_arn: {type: string, defaut: '', required: false} rw_sccache_bucket: {type: string, defaut: '', required: false} rw_sccache_region: {type: string, defaut: '', required: false} + sccache_dist_scheduler_url: {type: string, defaut: '', required: false} runs: using: composite @@ -32,3 +33,4 @@ runs: aws_role_arn: "${{ inputs.aws_role_arn }}" rw_sccache_bucket: "${{ inputs.rw_sccache_bucket }}" rw_sccache_region: "${{ inputs.rw_sccache_region }}" + sccache_dist_scheduler_url: "${{ inputs.sccache_dist_scheduler_url }}" diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 9f34995de..f531fa60e 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -35,24 +35,68 @@ jobs: contents: read pull-requests: read with: - arch: '["amd64"]' + arch: '["amd64", "arm64"]' cuda: '["12.8"]' - node_type: cpu32 + node_type: cpu8 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY + rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN build_command: | + # Repopulate the cache + export SCCACHE_RECACHE=1 + + # Prevent the sccache server from shutting down export SCCACHE_IDLE_TIMEOUT=0 - SCCACHE_NO_DAEMON=1 sccache --stop-server - sccache -z; + + # Infinitely retry transient errors + export SCCACHE_DIST_MAX_RETRIES=inf + + # Never fallback to locally compiling + export SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false + + # Disable CUB/Thrust arch-dependent namespaces. + # These prevent reusing ptx and cubins from multi-arch + # compilations in single-arch subset compilations. + for VAR in CFLAGS CXXFLAGS CUDAFLAGS; do + export "$VAR=${!VAR:+${!VAR} }-DCUB_DISABLE_NAMESPACE_MAGIC -DCUB_IGNORE_NAMESPACE_MAGIC_ERROR" + export "$VAR=${!VAR:+${!VAR} }-DTHRUST_DISABLE_ABI_NAMESPACE -DTHRUST_IGNORE_ABI_NAMESPACE_ERROR" + done + + # Install the latest sccache client + devcontainer-utils-install-sccache --repo trxcllnt/sccache; + + # Configure the sccache client to talk to the build cluster + devcontainer-utils-init-sccache-dist \ + --enable-sccache-dist \ + - <<< "--auth-token '$RAPIDS_AUX_SECRET_1' \ + --auth-type 'token' \ + "; + + # Verify sccache cache location sccache --show-adv-stats; - clone-all -j$(nproc) -v -q --clone-upstream --single-branch --shallow-submodules; - build-all \ - -v \ - -j$(nproc --ignore=1) \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_TESTS=ON \ - -DBUILD_BENCHMARKS=ON \ - -DBUILD_PRIMS_BENCH=ON \ - -DRAFT_COMPILE_LIBRARY=ON \ - -DBUILD_CUGRAPH_MG_TESTS=ON \ - ; + + # Clone all the repos + clone-all -j$(nproc) -v -q --clone-upstream --single-branch --shallow-submodules --no-update-env; + + # Create the python env without ninja. + # ninja -j10000 fails with `ninja: FATAL: pipe: Too many open files`. + # This appears to have been fixed 13 years ago (https://github.com/ninja-build/ninja/issues/233), + # so that fix needs to be integrated into the kitware pip ninja builds. + rapids-post-start-command --exclude <(echo ninja); + + # Configure all the C++ libs + configure-all \ + -j100000 \ + -Wno-dev \ + -DBUILD_TESTS=ON \ + -DBUILD_BENCHMARKS=ON \ + -DBUILD_PRIMS_BENCH=ON \ + -DBUILD_SHARED_LIBS=ON \ + -DRAFT_COMPILE_LIBRARY=ON \ + -DBUILD_CUGRAPH_MG_TESTS=ON \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5; + + # Build all the libs + build-all -j100000; + + # Print cache and dist stats sccache --show-adv-stats; diff --git a/.github/workflows/build-and-test-feature.yml b/.github/workflows/build-and-test-feature.yml index b15b78fe2..a27d57d88 100644 --- a/.github/workflows/build-and-test-feature.yml +++ b/.github/workflows/build-and-test-feature.yml @@ -40,3 +40,4 @@ jobs: aws_role_arn: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs' || '' }}" rw_sccache_bucket: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'rapids-sccache-devs' || '' }}" rw_sccache_region: "${{ vars.AWS_REGION }}" + sccache_dist_scheduler_url: "sccache.gha-runners.nvidia.com" diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json index fe3c2e314..eea08310e 100644 --- a/features/src/rapids-build-utils/devcontainer-feature.json +++ b/features/src/rapids-build-utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVIDIA RAPIDS devcontainer build utilities", "id": "rapids-build-utils", - "version": "25.6.1", + "version": "25.6.2", "description": "A feature to install the RAPIDS devcontainer build utilities", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh index ba7b6d410..5fe615818 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh @@ -63,7 +63,7 @@ generate_script() { } generate_all_script_impl() { - local bin="${SCRIPT}-all"; + local bin="${PREFIX:-${SCRIPT}}-${SUFFIX:-all}"; if test -n "${bin:+x}" && ! test -f "${TMP_SCRIPT_DIR}/${bin}"; then ( cat - \ @@ -378,8 +378,18 @@ generate_scripts() { NAME="${cloned_repos[0]:-${repo_names[0]:-}}" \ NAMES="${repo_names[*]@Q}" \ SCRIPT="${script}" \ + PREFIX="${script}" \ generate_all_script ; - done; + done + for kind in "cpp" "python"; do + # Generate a script to run a type of build for all repos + NAME="${cloned_repos[0]:-${repo_names[0]:-}}" \ + NAMES="${repo_names[*]@Q}" \ + SCRIPT="${kind}.build" \ + PREFIX="build" \ + SUFFIX="all-${kind}" \ + generate_all_script ; + done fi } diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh index 064396308..1809640e8 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh @@ -4,10 +4,6 @@ # rapids-get-num-archs-jobs-and-load [OPTION]... # # Compute an appropriate total number of jobs, load, and CUDA archs to build in parallel. -# This routine scales the input `-j` with respect to the `-a` and `-m` values, taking into account the -# amount of available system memory (free mem + swap), in order to balance the job and arch parallelism. -# -# note: This wouldn't be necessary if `nvcc` interacted with the POSIX jobserver. # # Boolean options: # -h,--help Print this text. @@ -18,16 +14,6 @@ # -j,--parallel Run parallel compilation jobs. # --max-archs Build at most CUDA archs in parallel. # (default: 3) -# --max-total-system-memory An upper-bound on the amount of total system memory (in GiB) to use during -# C++ and CUDA device compilations. -# Smaller values yield fewer parallel C++ and CUDA device compilations. -# (default: all available memory) -# --max-device-obj-memory-usage An upper-bound on the amount of memory each CUDA device object compilation -# is expected to take. This is used to estimate the number of parallel device -# object compilations that can be launched without hitting the system memory -# limit. -# Higher values yield fewer parallel CUDA device object compilations. -# (default: 1) # shellcheck disable=SC1091 . rapids-generate-docstring; @@ -41,10 +27,9 @@ get_num_archs_jobs_and_load() { # shellcheck disable=SC1091 . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'get-num-archs-jobs-and-load'; - # The return value of nproc is (who knew!) constrained by the - # values of OMP_NUM_THREADS and/or OMP_THREAD_LIMIT - # Since we want the physical number of processors here, pass --all - local -r n_cpus="$(nproc --all)"; + # nproc --all returns 2x the number of threads in Ubuntu24.04+, + # so instead we cound the number of processors in /proc/cpuinfo + local -r n_cpus="$(grep -cP 'processor\s+:' /proc/cpuinfo)"; if test ${#j[@]} -gt 0 && ! test -n "${j:+x}"; then j="${n_cpus}"; @@ -52,22 +37,12 @@ get_num_archs_jobs_and_load() { parallel="${j:-${JOBS:-${PARALLEL_LEVEL:-1}}}"; max_archs="${max_archs:-${MAX_DEVICE_OBJ_TO_COMPILE_IN_PARALLEL:-${arch:-}}}"; - max_device_obj_memory_usage="${max_device_obj_memory_usage:-${MAX_DEVICE_OBJ_MEMORY_USAGE:-1Gi}}"; - - local num_re="^[0-9]+$"; - - # Assume un-suffixed inputs means gibibytes - if [[ "${max_device_obj_memory_usage}" =~ ${num_re} ]]; then - max_device_obj_memory_usage="${max_device_obj_memory_usage}Gi"; - fi - - max_device_obj_memory_usage="$(numfmt --from=auto "${max_device_obj_memory_usage}")"; local n_arch="${archs:-1}"; - # currently: 70-real;75-real;80-real;86-real;90 - # see: https://github.com/rapidsai/rapids-cmake/blob/branch-24.04/rapids-cmake/cuda/set_architectures.cmake#L54 - local n_arch_rapids=5; + # currently: 70-real;75-real;80-real;86-real;90-real;100-real;120 + # see: https://github.com/rapidsai/rapids-cmake/blob/branch-25.04/rapids-cmake/cuda/set_architectures.cmake#L59 + local n_arch_rapids=7; if ! test -n "${archs:+x}" && test -n "${INFER_NUM_DEVICE_ARCHITECTURES:+x}"; then archs="$(rapids-select-cmake-define CMAKE_CUDA_ARCHITECTURES "${OPTS[@]}" || echo)"; @@ -101,31 +76,8 @@ get_num_archs_jobs_and_load() { n_arch=$((n_arch > max_archs ? max_archs : n_arch)); fi - local mem_for_device_objs="$((n_arch * max_device_obj_memory_usage))"; - local mem_total="${max_total_system_memory:-${MAX_TOTAL_SYSTEM_MEMORY:-}}"; - - if ! test -n "${mem_total:+x}"; then - local -r free_mem="$(free --bytes | grep -E '^Mem:' | tr -s '[:space:]' | cut -d' ' -f7 || echo '0')"; - local -r freeswap="$(free --bytes | grep -E '^Swap:' | tr -s '[:space:]' | cut -d' ' -f4 || echo '0')"; - mem_total="$((free_mem + freeswap))"; - # Assume un-suffixed inputs means gibibytes - elif [[ "${mem_total}" =~ ${num_re} ]]; then - mem_total="${mem_total}Gi"; - fi - mem_total="$(numfmt --from=auto "${mem_total}")"; - local n_load=$((parallel > n_cpus ? n_cpus : parallel)); - # shellcheck disable=SC2155 - local n_jobs="$( - echo " -scale=0 -max_cpu=(${n_load} / ${n_arch} / 2 * 3) -max_mem=(${mem_total} / ${mem_for_device_objs}) -if(max_cpu < max_mem) max_cpu else max_mem -" | bc - )" - n_jobs=$((n_jobs < 1 ? 1 : n_jobs)); - n_jobs=$((n_arch > 1 ? n_jobs : n_load)); + local n_jobs="$((parallel < 1 ? 1 : parallel))"; echo "declare n_arch=${n_arch}"; echo "declare n_jobs=${n_jobs}"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.cpp.build.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.cpp.build.tmpl.sh new file mode 100755 index 000000000..f5f8f52b5 --- /dev/null +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.cpp.build.tmpl.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Usage: +# build-all-cpp [OPTION]... +# +# Runs build--cpp for each repo in ${NAMES}. +# +# Forwards relevant arguments to each underlying script. +# +# @_include_options /usr/bin/build-${NAME}-cpp -h | tail -n+2; + +# shellcheck disable=SC1091 +. rapids-generate-docstring; + +_build_all_cpp() { + local -; + set -euo pipefail; + + eval "$(_parse_args --take '-h,--help' "$@" <&0)"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'build-all build-all-cpp'; + + for name in ${NAMES}; do + if command -v build-${name}-cpp >/dev/null 2>&1; then + build-${name}-cpp "${OPTS[@]}"; + fi + done +} + +_build_all_cpp "$@" <&0; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.python.build.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.python.build.tmpl.sh new file mode 100755 index 000000000..3690da880 --- /dev/null +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.python.build.tmpl.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Usage: +# build-all-python [OPTION]... +# +# Runs build--python for each repo in ${NAMES}. +# +# Forwards relevant arguments to each underlying script. +# +# @_include_options /usr/bin/build-${NAME}-python -h | tail -n+2; + +# shellcheck disable=SC1091 +. rapids-generate-docstring; + +_build_all_python() { + local -; + set -euo pipefail; + + eval "$(_parse_args --take '-h,--help' "$@" <&0)"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'build-all build-all-python'; + + for name in ${NAMES}; do + if command -v build-${name}-python >/dev/null 2>&1; then + build-${name}-python "${OPTS[@]}"; + fi + done +} + +_build_all_python "$@" <&0; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh index 83e659dc1..650976a12 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh @@ -57,6 +57,7 @@ configure_${CPP_LIB}_cpp() { time ( export ${CPP_ENV} PATH="$PATH"; + SCCACHE_NO_DIST_COMPILE=1 \ CUDAFLAGS="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}" \ cmake "${cmake_args[@]}"; { set +x; } 2>/dev/null; echo -n "lib${CPP_LIB} configure time:"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh index b0c35ce88..cf99dea91 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh @@ -82,6 +82,23 @@ build_${PY_LIB}_python_wheel() { pip_args+=("${PY_SRC}"); + # Ensure SCCACHE_NO_DIST_COMPILE=1 is set while configuring + # so CMake's compiler tests never use the build cluster. + + if ! test -f /tmp/sccache_no_dist_compile.cmake; then + cat <<"EOF" > /tmp/sccache_no_dist_compile.cmake +set(ENV{SCCACHE_NO_DIST_COMPILE} "1") +EOF + fi + + # Merge with outer `-DCMAKE_PROJECT_INCLUDE_BEFORE=` if provided + local -a cmake_project_include_before="( + $(rapids-select-cmake-define CMAKE_PROJECT_INCLUDE_BEFORE "$@" || echo) + /tmp/sccache_no_dist_compile.cmake + )"; + # Join with semicolons + cmake_args+=("-DCMAKE_PROJECT_INCLUDE_BEFORE=$(IFS=";"; echo "${cmake_project_include_before[*]}")") + trap "rm -rf '${PY_SRC}/${py_lib//"-"/"_"}.egg-info'" EXIT; time ( diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh index e91d23647..9af3fc826 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh @@ -100,6 +100,23 @@ install_${PY_LIB}_python() { pip_args+=("${PY_SRC}"); fi + # Ensure SCCACHE_NO_DIST_COMPILE=1 is set while configuring + # so CMake's compiler tests never use the build cluster. + + if ! test -f /tmp/sccache_no_dist_compile.cmake; then + cat <<"EOF" > /tmp/sccache_no_dist_compile.cmake +set(ENV{SCCACHE_NO_DIST_COMPILE} "1") +EOF + fi + + # Merge with outer `-DCMAKE_PROJECT_INCLUDE_BEFORE=` if provided + local -a cmake_project_include_before="( + $(rapids-select-cmake-define CMAKE_PROJECT_INCLUDE_BEFORE "$@" || echo) + /tmp/sccache_no_dist_compile.cmake + )"; + # Join with semicolons + cmake_args+=("-DCMAKE_PROJECT_INCLUDE_BEFORE=$(IFS=";"; echo "${cmake_project_include_before[*]}")") + trap "rm -rf '${PY_SRC}/${py_lib//"-"/"_"}.egg-info'" EXIT; time ( diff --git a/features/src/utils/devcontainer-feature.json b/features/src/utils/devcontainer-feature.json index b078fc120..ee6251c28 100644 --- a/features/src/utils/devcontainer-feature.json +++ b/features/src/utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "devcontainer-utils", "id": "utils", - "version": "25.6.2", + "version": "25.6.3", "description": "A feature to install RAPIDS devcontainer utility scripts", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/utils/install.sh b/features/src/utils/install.sh index 514c9b56e..dfb691f91 100644 --- a/features/src/utils/install.sh +++ b/features/src/utils/install.sh @@ -14,6 +14,7 @@ PKGS=( sudo wget socat + procps tzdata gettext-base openssh-client @@ -104,6 +105,12 @@ declare -a commands_and_sources=( "init-gitlab-cli gitlab/cli/init.sh" "clone-gitlab-repo gitlab/repo/clone.sh" "print-missing-gitlab-token-warning gitlab/print-missing-token-warning.sh" + "install-sccache sccache/install.sh" + "start-sccache sccache/start.sh" + "stop-sccache sccache/stop.sh" + "init-sccache-dist sccache/dist/init.sh" + "configure-sccache-dist sccache/dist/configure.sh" + "sccache-dist-status sccache/dist/status.sh" ) # Install alternatives diff --git a/features/src/utils/opt/devcontainer/bin/creds/s3/propagate.sh b/features/src/utils/opt/devcontainer/bin/creds/s3/propagate.sh index 1410b7e37..ec0129cd9 100755 --- a/features/src/utils/opt/devcontainer/bin/creds/s3/propagate.sh +++ b/features/src/utils/opt/devcontainer/bin/creds/s3/propagate.sh @@ -13,13 +13,11 @@ _creds_s3_propagate() { local num_restarts="0"; - if test -n "$(pgrep sccache || echo)"; then - sccache --stop-server >/dev/null 2>&1 || true; - fi + devcontainer-utils-stop-sccache --kill-all; while true; do - if sccache --start-server >/dev/null 2>&1; then + if devcontainer-utils-start-sccache >/dev/null; then if [ "${num_restarts}" -gt "0" ]; then echo "Success!"; fi exit 0; fi diff --git a/features/src/utils/opt/devcontainer/bin/creds/s3/test.sh b/features/src/utils/opt/devcontainer/bin/creds/s3/test.sh index f0e075558..006fa6a37 100755 --- a/features/src/utils/opt/devcontainer/bin/creds/s3/test.sh +++ b/features/src/utils/opt/devcontainer/bin/creds/s3/test.sh @@ -28,17 +28,14 @@ _creds_s3_test() { local aws_session_token="${AWS_SESSION_TOKEN:-"$(sed -n 's/aws_session_token=//p' ~/.aws/credentials 2>/dev/null)"}"; local aws_secret_access_key="${AWS_SECRET_ACCESS_KEY:-"$(sed -n 's/aws_secret_access_key=//p' ~/.aws/credentials 2>/dev/null)"}"; - if test -n "$(pgrep sccache || echo)"; then - sccache --stop-server >/dev/null 2>&1 || true; - fi - - SCCACHE_BUCKET="${bucket:-}" \ - SCCACHE_REGION="${region:-}" \ - AWS_ACCESS_KEY_ID="${aws_access_key_id:-}" \ - AWS_SESSION_TOKEN="${aws_session_token:-}" \ + AWS_PROFILE=none \ + SCCACHE_BUCKET="${bucket:-}" \ + SCCACHE_REGION="${region:-}" \ + AWS_ACCESS_KEY_ID="${aws_access_key_id:-}" \ + AWS_SESSION_TOKEN="${aws_session_token:-}" \ AWS_SECRET_ACCESS_KEY="${aws_secret_access_key:-}" \ - sccache --start-server >/dev/null 2>&1; - sccache --show-stats | grep -qE 'Cache location \s+ s3'; + devcontainer-utils-start-sccache >/dev/null \ + && sccache --show-stats 2>/dev/null | grep -qE 'Cache location \s+ s3'; } _creds_s3_test "$@"; diff --git a/features/src/utils/opt/devcontainer/bin/github/cli/init.sh b/features/src/utils/opt/devcontainer/bin/github/cli/init.sh index 85cb6031a..94808646f 100755 --- a/features/src/utils/opt/devcontainer/bin/github/cli/init.sh +++ b/features/src/utils/opt/devcontainer/bin/github/cli/init.sh @@ -31,7 +31,7 @@ init_github_cli() { | tr -d ',' \ )"; - local needed_scopes="read:org"; + local needed_scopes="read:org read:enterprise"; needed_scopes="$( \ comm -23 \ diff --git a/features/src/utils/opt/devcontainer/bin/post-attach-command.sh b/features/src/utils/opt/devcontainer/bin/post-attach-command.sh index 1029ef502..700b99350 100755 --- a/features/src/utils/opt/devcontainer/bin/post-attach-command.sh +++ b/features/src/utils/opt/devcontainer/bin/post-attach-command.sh @@ -1,8 +1,34 @@ #! /usr/bin/env bash if ! test -n "${SKIP_DEVCONTAINER_UTILS_POST_ATTACH_COMMAND:+x}"; then + sudo mkdir -m 0777 -p /var/log/devcontainer-utils; + sudo touch /var/log/devcontainer-utils/creds-s3.log; + sudo chmod 0777 /var/log/devcontainer-utils/creds-s3.log; + # shellcheck disable=SC1091 . devcontainer-utils-init-git-interactive; + + # Reset sccache-dist configuration + devcontainer-utils-init-sccache-dist; + + # Initialize the GitHub CLI with the appropriate user scopes # shellcheck disable=SC1091 . devcontainer-utils-creds-s3-init; + + # Update sccache client configuration to enable/disable sccache-dist + if test -n "${DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST:+x}"; then + if test -n "${SCCACHE_DIST_TOKEN:+x}"; then + devcontainer-utils-install-sccache --repo trxcllnt/sccache; + # Update ~/.config/sccache/config to use $SCCACHE_DIST_TOKEN + devcontainer-utils-init-sccache-dist \ + --enable-sccache-dist - <<< " \ + --auth-type 'token' \ + --auth-token '${SCCACHE_DIST_TOKEN}' \ + "; + elif gh auth status >/dev/null 2>&1; then + devcontainer-utils-install-sccache --repo trxcllnt/sccache; + # Update ~/.config/sccache/config to use gh token auth + devcontainer-utils-init-sccache-dist --enable-with-github-auth; + fi + fi fi diff --git a/features/src/utils/opt/devcontainer/bin/sccache/dist/configure.sh b/features/src/utils/opt/devcontainer/bin/sccache/dist/configure.sh new file mode 100755 index 000000000..319d87857 --- /dev/null +++ b/features/src/utils/opt/devcontainer/bin/sccache/dist/configure.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash + +# Usage: +# devcontainer-utils-configure-sccache-dist [OPTION]... +# +# Generate and write sccache-dist configuration to `$HOME/.config/sccache/config` +# +# Boolean options: +# -h,--help Print this text. +# --no-local-compile-fallback Disable building locally after retrying transient sccache-dist errors. +# (default: false) +# --use-connection-pool Enable sccache client HTTP connection pool. +# (default: false) +# +# Options that require values: +# --auth-type (token|oauth2_code_grant_pkce|oauth2_implicit) Auth type for build cluster auth. +# Disables client auth if omitted (not recommended). +# --auth-token Token used to authenticate with the build cluster when `--auth-type=token`. +# --client-id Client ID used to authenticate with the build cluster when `--auth-type=(oauth2_code_grant_pkce|oauth2_implicit)`. +# --auth-url Auth URL used to authenticate with the build cluster when `--auth-type=(oauth2_code_grant_pkce|oauth2_implicit)`. +# --token-url Token URL used to authenticate with the build cluster when `--auth-type=oauth2_code_grant_pkce`. +# --connect-timeout The sccache client HTTP connection timeout. +# (default: 30) +# --request-timeout The sccache client HTTP request timeout. +# (default: 1800) +# --scheduler-url URL of the sccache-dist build cluster. +# --max-retries Maximum number of times to retry transient sccache-dist errors. +# Pass `--max-retries inf` to retry infinitely. +# Combining `--max-retries inf` with `--no-local-compile-fallback` +# ensures the sccache client relies exclusively on the build cluster. +# (default: 0) + +_configure_sccache_dist() { + local -; + set -euo pipefail; + + eval "$(devcontainer-utils-parse-args "$0" "$@" <&0)"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'sccache configure-sccache-dist'; + + auth_type="${auth_type-}"; + max_retries="${max_retries:-${SCCACHE_DIST_MAX_RETRIES-}}"; + scheduler_url="${scheduler_url:-${SCCACHE_DIST_SCHEDULER_URL-}}"; + connect_timeout="${connect_timeout:-${SCCACHE_DIST_CONNECT_TIMEOUT-}}"; + request_timeout="${request_timeout:-${SCCACHE_DIST_REQUEST_TIMEOUT-}}"; + + local connection_pool="${SCCACHE_DIST_CONNECTION_POOL-}"; + + if test -n "${use_connection_pool:+x}"; then + connection_pool="true"; + fi + + local fallback_to_local_compile="${SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE-}"; + + if test -n "${no_local_compile_fallback:+x}"; then + fallback_to_local_compile="false"; + fi + + local sccache_conf="${SCCACHE_CONF:-"$HOME/.config/sccache/config"}"; + + mkdir -p "$(dirname "$sccache_conf")"; + touch "$sccache_conf"; + + # Remove the values we're about to set. Deletes toml ranges like this: + # [dist] + # max_retries = 0 + # fallback_to_local_compile = true + # scheduler_url = "foo.bar.xyz.com" + # + # [dist.net] + # connection_pool = true + # connect_timeout = 10 + # request_timeout = 600 + # + # [dist.auth] + # type = "foo" + # token = "bar" + sed -Ei '/^\[dist.*\]/,/^(\s|\[)*$/d' "${sccache_conf}"; + + # Write our new values + if ! grep -qE "^$" <<< "${scheduler_url}"; then + cat <<< "[dist]" >> "${sccache_conf}"; + cat <<< "scheduler_url = \"${scheduler_url}\"" >> "${sccache_conf}"; + if ! grep -qE "^$" <<< "${max_retries}"; then + cat <<< "max_retries = ${max_retries}" >> "${sccache_conf}"; + fi + if ! grep -qE "^$" <<< "${fallback_to_local_compile}"; then + cat <<< "fallback_to_local_compile = ${fallback_to_local_compile}" >> "${sccache_conf}"; + fi + + cat <<< "[dist.net]" >> "${sccache_conf}"; + if ! grep -qE "^$" <<< "${connection_pool}"; then + cat <<< "connection_pool = ${connection_pool}" >> "${sccache_conf}"; + fi + if ! grep -qE "^$" <<< "${connect_timeout}"; then + cat <<< "connect_timeout = "${connect_timeout}"" >> "${sccache_conf}"; + fi + if ! grep -qE "^$" <<< "${request_timeout}"; then + cat <<< "request_timeout = "${request_timeout}"" >> "${sccache_conf}"; + fi + + if test "${auth_type}" = token; then + if ! grep -qE "^$" <<< "${auth_token-}"; then + cat <<< "[dist.auth]" >> "${sccache_conf}"; + cat <<< "type = \"${auth_type}\"" >> "${sccache_conf}"; + cat <<< "token = \"${auth_token}\"" >> "${sccache_conf}"; + fi + elif test "${auth_type}" = oauth2_implicit; then + if ! grep -qE "^$" <<< "${auth_url-}" \ + && ! grep -qE "^$" <<< "${client_id-}"; then + cat <<< "[dist.auth]" >> "${sccache_conf}"; + cat <<< "type = \"${auth_type}\"" >> "${sccache_conf}"; + cat <<< "auth_url = \"${auth_url}\"" >> "${sccache_conf}"; + cat <<< "client_id = \"${client_id}\"" >> "${sccache_conf}"; + fi + elif test "${auth_type}" = oauth2_code_grant_pkce; then + if ! grep -qE "^$" <<< "${auth_url-}" \ + && ! grep -qE "^$" <<< "${client_id-}" \ + && ! grep -qE "^$" <<< "${token_url-}"; then + cat <<< "[dist.auth]" >> "${sccache_conf}"; + cat <<< "type = \"${auth_type}\"" >> "${sccache_conf}"; + cat <<< "auth_url = \"${auth_url}\"" >> "${sccache_conf}"; + cat <<< "client_id = \"${client_id}\"" >> "${sccache_conf}"; + cat <<< "token_url = \"${token_url}\"" >> "${sccache_conf}"; + fi + fi + fi +} + +_configure_sccache_dist "$@" <&0; diff --git a/features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh b/features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh new file mode 100755 index 000000000..624656407 --- /dev/null +++ b/features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +# Usage: +# devcontainer-utils-init-sccache-dist [OPTION]... +# +# Generate and write sccache-dist configuration to `$HOME/.config/sccache/config`, +# then restart sccache to ensure the configuration is applied. +# +# Boolean options: +# -h,--help Print this text. +# -e,--enable-sccache-dist Enable sccache-dist. If omitted, disable sccache-dist. +# (default: false) +# --enable-with-github-auth Enable sccache-dist and configure it to use GitHub token auth. +# Shorthand for `-e --auth-type token --auth-token $(gh auth token)` +# (default: false) +# --no-local-compile-fallback Disable building locally after retrying transient sccache-dist errors. +# (default: false) +# --use-connection-pool Enable sccache client HTTP connection pool. +# (default: false) +# +# Options that require values: +# --auth-type (token|oauth2_code_grant_pkce|oauth2_implicit) Auth type for build cluster auth. +# (default: token) +# --auth-token Token used to authenticate with the build cluster when `--auth-type=token`. +# --client-id Client ID used to authenticate with the build cluster when `--auth-type=(oauth2_code_grant_pkce|oauth2_implicit)`. +# --auth-url Auth URL used to authenticate with the build cluster when `--auth-type=(oauth2_code_grant_pkce|oauth2_implicit)`. +# --token-url Token URL used to authenticate with the build cluster when `--auth-type=oauth2_code_grant_pkce`. +# --connect-timeout The sccache client HTTP connection timeout. +# (default: 30) +# --request-timeout The sccache client HTTP request timeout. +# (default: 1800) +# --scheduler-url URL of the sccache-dist build cluster. +# --max-retries Maximum number of times to retry transient sccache-dist errors. +# Pass `--max-retries inf` to retry infinitely. +# Combining `--max-retries inf` with `--no-local-compile-fallback` +# ensures the sccache client relies exclusively on the build cluster. +# (default: 0) + +_init_sccache_dist() { + local -; + set -euo pipefail; + + eval "$(devcontainer-utils-parse-args "$0" --take '-e,--enable-sccache-dist --enable-with-github-auth' "$@" <<< "")"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'sccache init-sccache-dist'; + + if test -n "${enable_sccache_dist:+x}"; then + # Add sccache-dist configuration to ~/.config/sccache/config + devcontainer-utils-configure-sccache-dist "${OPTS[@]}" - <&0; + elif test -n "${enable_with_github_auth:+x}"; then + devcontainer-utils-configure-sccache-dist - <<< " \ + --auth-type 'token' --auth-token '$(gh auth token)'"; + else + # Delete sccache-dist configuration from ~/.config/sccache/config + SCCACHE_DIST_MAX_RETRIES= \ + SCCACHE_DIST_SCHEDULER_URL= \ + SCCACHE_DIST_CONNECT_TIMEOUT= \ + SCCACHE_DIST_REQUEST_TIMEOUT= \ + SCCACHE_DIST_CONNECTION_POOL= \ + SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE= \ + devcontainer-utils-configure-sccache-dist; + fi + + # Restart the sccache client with the new configuration + devcontainer-utils-start-sccache; + + # Verify sccache-dist status and configuration + if sccache --dist-status 2>/dev/null | jq -er '.SchedulerStatus? != null' >/dev/null 2>&1; then + devcontainer-utils-sccache-dist-status -c 17 -f tsv; + fi +} + +_init_sccache_dist "$@" <&0; diff --git a/features/src/utils/opt/devcontainer/bin/sccache/dist/status.sh b/features/src/utils/opt/devcontainer/bin/sccache/dist/status.sh new file mode 100755 index 000000000..dead1e8ff --- /dev/null +++ b/features/src/utils/opt/devcontainer/bin/sccache/dist/status.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +# Usage: +# devcontainer-utils-sccache-dist-status [OPTION]... +# +# Print and optionally format `sccache --dist-status`. +# +# Boolean options: +# -h,--help Print this text. +# +# Options that require values: +# -c|--col-width Max column width in number of characters. +# String columns wider than this will be truncated with "...". +# (default: $COLUMNS) +# -f|--format (csv|tsv|json) The `sccache --dist-status` output format. +# (default: "json") +# + +_sccache_dist_status() { + + local -; + set -euo pipefail; + + eval "$(devcontainer-utils-parse-args "$0" "$@" <&0)"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'sccache sccache-dist-status'; + + f="${f:-${format:-json}}"; + c="${c:-${col_width:-${COLUMNS:-1000000000}}}"; + + # Print current dist status to verify we're connected + sccache 2>/dev/null --dist-status \ + | { + # Passthrough if the format is json + if test "$f" = json; then + cat - <(echo) + else + + cat - | jq -r -f <(cat < $c) + then \$x[0:$((c-3))] + "..." + else \$x + end + ); + def info_to_row: { + time: now | floor, + type: (.type // "server"), + id: .id, + servers: (if .servers == null then "-" else (.servers | length) end), + cpus: .info.occupancy, + util: ((.info.cpu_usage // 0) * 100 | round | . / 100 | tostring | . + "%"), + jobs: (.jobs.loading + .jobs.pending + .jobs.running), + loading: .jobs.loading, + pending: .jobs.pending, + running: .jobs.running, + accepted: .jobs.accepted, + finished: .jobs.finished, + u_time: ((.u_time // 0) | tostring | . + "s") + }; + + .SchedulerStatus as [\$x, \$y] | [ + (\$y + { id: \$x, type: "scheduler", u_time: (\$y.servers // {} | map(.u_time) | min | . // "-" | tostring) }), + (\$y.servers // [] | sort_by(.id)[]) + ] + | map(info_to_row) as \$rows + | (\$rows[0] | keys_unsorted) as \$cols + | (\$rows | map(. as \$row | \$cols | map(\$row[.] | truncate_val))) as \$rows + | (\$cols | map(truncate_val)), \$rows[] | @csv +EOF +) + fi + } \ + | { + # Passthrough if the format is csv or json + # Otherwise, transform the csv into a tsv. + if test "$f" = tsv; then + if [[ "$(grep DISTRIB_RELEASE= /etc/lsb-release | cut -d= -f2)" > "20.04" ]]; then + cat - | sed 's/\"//g' | column -t -s, -R $(seq -s, 1 13) + else + cat - | sed 's/\"//g' | column -t -s, + fi + else + cat - + fi + } +} + +_sccache_dist_status "$@" <&0; diff --git a/features/src/utils/opt/devcontainer/bin/sccache/install.sh b/features/src/utils/opt/devcontainer/bin/sccache/install.sh new file mode 100755 index 000000000..4a650b862 --- /dev/null +++ b/features/src/utils/opt/devcontainer/bin/sccache/install.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +# Usage: +# devcontainer-utils-install-sccache [OPTION]... +# +# Install a specific sccache client binary from GitHub releases. +# +# Boolean options: +# -h,--help Print this text. +# +# Options that require values: +# --repo The GitHub repository to use. +# (default: mozilla/sccache) +# --version The sccache version to install (e.g. `0.10.0`). +# (default: latest) +# + + +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + +# Figure out correct version of a three part version number is not passed +_find_version_from_git_tags() { + local variable_name="$1" + local requested_version="${!variable_name}" + if [ "${requested_version}" = "none" ]; then return; fi + local repository="$2" + local prefix="${3:-"tags/v"}" + local separator="${4:-"."}" + local suffix="${5:-}" + local last_part_optional="${6:-"false"}" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part= + if [ "${last_part_optional}" = "true" ]; then + last_part+="(${escaped_separator}[0-9]+)?" + last_part+="(${escaped_separator}[0-9]+)?" + if [ -n "${suffix}" ]; then + last_part+="(${suffix})?" + fi + else + last_part+="${escaped_separator}[0-9]+" + last_part+="${escaped_separator}[0-9]+" + if [ -n "${suffix}" ]; then + last_part+="(${suffix})" + fi + fi + local regex="${prefix}\\K[0-9]+${last_part}$" + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch + fi + local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + fi + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + _upvar "${variable_name}" "$(head -n 1 <<< "${version_list}")" + else + set +e + _upvar "${variable_name}" "$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! grep "^${!variable_name//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + return 1 + fi +} + +_install_sccache() { + local -; + set -euo pipefail; + + eval "$(devcontainer-utils-parse-args "$0" "$@" <&0)"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'sccache install-sccache'; + + local sccache_version="${version:-"latest"}"; + local github_repo="${repo:-"mozilla/sccache"}"; + + _find_version_from_git_tags sccache_version "https://github.com/$github_repo" "" "" "-.*" "true"; + + while pgrep sccache >/dev/null 2>&1; do + devcontainer-utils-stop-sccache --kill-all; + done + + # Install sccache + wget --no-hsts -q -O- \ + "https://github.com/$github_repo/releases/download/v$sccache_version/sccache-v$sccache_version-$(uname -m)-unknown-linux-musl.tar.gz" \ + | sudo tar -C /usr/bin -zf - --wildcards --strip-components=1 -x '*/sccache' \ + && sudo chmod +x /usr/bin/sccache; + + echo "Installed sccache v$(sccache --version | cut -d' ' -f2) to $(which sccache)" +} + +_install_sccache "$@" <&0; diff --git a/features/src/utils/opt/devcontainer/bin/sccache/start.sh b/features/src/utils/opt/devcontainer/bin/sccache/start.sh new file mode 100755 index 000000000..905a5286b --- /dev/null +++ b/features/src/utils/opt/devcontainer/bin/sccache/start.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +# Usage: +# devcontainer-utils-start-sccache [OPTION]... +# +# Start the sccache server in the foreground or background. +# +# Boolean options: +# -h,--help Print this text. +# -f,--foreground Start the sccache server in the foreground. +# +# Options that require values: +# -p,--port Start the sccache server on . +# If there is already a server listening +# on this port, it will be killed first. +# (default: ${SCCACHE_SERVER_PORT:-4226}) +# + +_start_sccache() { + local -; + set -euo pipefail; + + eval "$(devcontainer-utils-parse-args "$0" "$@" <&0)"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'sccache start-sccache'; + + local sccache_port="${p:-${port:-${SCCACHE_SERVER_PORT:-4226}}}"; + local pidfile="/tmp/sccache.${sccache_port}.pid"; + + # Stop any existing server + devcontainer-utils-stop-sccache -p "${sccache_port}"; + + local logfile="${SCCACHE_ERROR_LOG:-/tmp/sccache.log}"; + local log_lvl="${SCCACHE_LOG:-${SCCACHE_SERVER_LOG-}}"; + + logfile="$(dirname "$logfile")/$(basename -s .log "$logfile").${sccache_port}.log"; + + if test -n "${f:-${foreground:+x}}"; then + # Unset this so sccache outputs to stderr + unset SCCACHE_ERROR_LOG; + # Increase the open file limit so users can do `make -j1024` + ulimit -n "$(ulimit -Hn)"; + # Start the sccache server in the foreground + RUST_LOG_STYLE="always" \ + SCCACHE_NO_DAEMON=1 \ + SCCACHE_START_SERVER=1 \ + SCCACHE_SERVER_LOG="${log_lvl}" \ + SCCACHE_SERVER_PORT="${sccache_port}" \ + sccache 2>&1 \ + | tee >(cat <(echo "=== sccache logfile: $logfile ===") \ + <(cat - ) \ + >&2) \ + >(sed -u 's/\x1B\[[0-9;]*[JKmsu]//g' >"$logfile") \ + >/dev/null ; + else + # Start the sccache server in the background + RUST_LOG_STYLE="never" \ + SCCACHE_ERROR_LOG="${logfile}" \ + SCCACHE_SERVER_LOG="${log_lvl}" \ + SCCACHE_SERVER_PORT="${sccache_port}" \ + sccache --start-server 2>/dev/null \ + | tee "$logfile"; + # Write the pid to the pidfile + pgrep sccache | sort -n | head -n1 | tee "${pidfile}" >/dev/null; + # Increase the open file limit so users can do `make -j1024` + prlimit --nofile=$(ulimit -Hn):$(ulimit -Hn) --pid "$(cat "${pidfile}")"; + echo "=== sccache logfile: $logfile ===" >&2; + echo "=== sccache pidfile: $pidfile ===" >&2; + fi +} + +_start_sccache "$@" <&0; diff --git a/features/src/utils/opt/devcontainer/bin/sccache/stop.sh b/features/src/utils/opt/devcontainer/bin/sccache/stop.sh new file mode 100755 index 000000000..39c54a81b --- /dev/null +++ b/features/src/utils/opt/devcontainer/bin/sccache/stop.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +# Usage: +# devcontainer-utils-stop-sccache [OPTION]... +# +# Gracefully or forcefully stop the sccache server. +# +# Boolean options: +# -h,--help Print this text. +# -k,--kill SIGKILL the sccache server. +# -a,--kill-all SIGKILL all sccache processes. +# +# Options that require values: +# -p,--port Stop the sccache server on . +# (default: ${SCCACHE_SERVER_PORT:-4226}) +# + +_stop_sccache() { + local -; + set -euo pipefail; + + eval "$(devcontainer-utils-parse-args "$0" "$@" <&0)"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'sccache stop-sccache'; + + if ! pgrep sccache >/dev/null 2>&1; then + return 0; + fi + + local sccache_port="${p:-${port:-${SCCACHE_SERVER_PORT:-4226}}}"; + local pidfile="/tmp/sccache.${sccache_port}.pid"; + + if test -n "${a:-${kill_all:+x}}"; then + # Shutdown all sccache processes forcefully + pkill -9 sccache >/dev/null 2>&1 || true; + elif test -n "${k:-${kill:+x}}" && test -f "${pidfile}"; then + # Shutdown the sccache process on `$sccache_port` forcefully + pkill -9 --pidfile "${pidfile}" >/dev/null 2>&1 || true; + else + # Shutdown gracefully + SCCACHE_SERVER_PORT="${sccache_port}" \ + sccache --stop-server >/dev/null 2>&1 || true; + if test -f "${pidfile}"; then + # Wait for the server to shutdown + if command -v pidwait >/dev/null 2>&1; then + pidwait --pidfile "${pidfile}" >/dev/null 2>&1 || true; + else + while IFS= read -r pid; do + if test -n "${pid:+x}"; then + while test -e "/proc/${pid}"; do + sleep 0.1; + done + fi + done < "${pidfile}" + fi + fi + fi + + rm -f "${pidfile}" 2>/dev/null || true; +} + +_stop_sccache "$@" <&0; diff --git a/features/test/utils/scenarios.json b/features/test/utils/scenarios.json index d26cbefd1..8bbb74684 100644 --- a/features/test/utils/scenarios.json +++ b/features/test/utils/scenarios.json @@ -8,7 +8,8 @@ "aws_role_arn": "${localEnv:aws_role_arn}", "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}", "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}", - "rw_sccache_region": "${localEnv:rw_sccache_region}" + "rw_sccache_region": "${localEnv:rw_sccache_region}", + "sccache_dist_scheduler_url": "${localEnv:sccache_dist_scheduler_url}" }, "features": { "ghcr.io/devcontainers/features/common-utils:2": { @@ -53,7 +54,8 @@ "aws_role_arn": "${localEnv:aws_role_arn}", "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}", "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}", - "rw_sccache_region": "${localEnv:rw_sccache_region}" + "rw_sccache_region": "${localEnv:rw_sccache_region}", + "sccache_dist_scheduler_url": "${localEnv:sccache_dist_scheduler_url}" }, "features": { "ghcr.io/devcontainers/features/common-utils:2": { @@ -98,7 +100,8 @@ "aws_role_arn": "${localEnv:aws_role_arn}", "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}", "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}", - "rw_sccache_region": "${localEnv:rw_sccache_region}" + "rw_sccache_region": "${localEnv:rw_sccache_region}", + "sccache_dist_scheduler_url": "${localEnv:sccache_dist_scheduler_url}" }, "features": { "ghcr.io/devcontainers/features/common-utils:2": { diff --git a/features/test/utils/ubuntu18.04.sh b/features/test/utils/ubuntu18.04.sh index d260ea905..581a4ecd6 100644 --- a/features/test/utils/ubuntu18.04.sh +++ b/features/test/utils/ubuntu18.04.sh @@ -18,6 +18,7 @@ source dev-container-features-test-lib export VAULT_S3_TTL="${VAULT_S3_TTL:-"900"}"; cp ~/.bashrc /tmp/.bashrc-clean; +sudo cp /usr/bin/sccache{,.orig}; utils_profile_script="$(find /etc/profile.d/ -type f -name '*-devcontainer-utils.sh')"; @@ -49,19 +50,24 @@ reset_state() { unset AWS_SESSION_TOKEN; unset AWS_SECRET_ACCESS_KEY; - rm -rf ~/.aws/ ~/.config/gh/; + while pgrep sccache >/dev/null 2>&1; do + devcontainer-utils-stop-sccache --kill-all; + done + + sudo rm -rf \ + ~/.aws/ \ + ~/.config/gh/ \ + ~/.config/sccache/config \ + /var/log/devcontainer-utils/creds-s3.log; cp /tmp/.bashrc-clean ~/.bashrc; + sudo cp /usr/bin/sccache{.orig,}; echo "#! /usr/bin/env bash" | sudo tee "${utils_profile_script}" >/dev/null; sudo chmod +x "${utils_profile_script}"; . ~/.bashrc; - - if test -n "$(pgrep sccache || echo)"; then - sccache --stop-server >/dev/null 2>&1 || true; - fi } write_bad_creds() { - devcontainer-utils-vault-s3-creds-persist <<< " + devcontainer-utils-creds-s3-persist <<< " --stamp='$(date '+%s')' --bucket='${rw_sccache_bucket:-}' --region='${rw_sccache_region:-}' @@ -83,6 +89,14 @@ expect_local_disk_cache_is_used() { grep -qE 'Cache location \s+ Local disk' <<< "${stats}"; } +expect_sccache_dist_auth_token_is_gh_token() { + set -e; + test -f ~/.config/sccache/config; + grep -qE "[dist.auth]" ~/.config/sccache/config; + grep -qE "type = \"token\"" ~/.config/sccache/config; + grep -qE "token = \"${gh_token}\"" ~/.config/sccache/config; +} + if test -n "${vault_host:+x}" \ && test -n "${rw_sccache_bucket:+x}"; then @@ -245,6 +259,44 @@ if test -n "${gh_token:+x}" \ check "bad stored creds with GH_TOKEN, AWS_ROLE_ARN, and SCCACHE_BUCKET should regenerate credentials" bad_stored_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_regenerate_credentials; fi +if test -n "${gh_token:-}" \ +&& test -n "${sccache_dist_scheduler_url:-}"; then + + does_not_set_sccache_dist_auth_config() { + reset_state; + DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ + SCCACHE_DIST_SCHEDULER_URL="${sccache_dist_scheduler_url}" \ + devcontainer-utils-post-attach-command; + if expect_sccache_dist_auth_token_is_gh_token; then + return 1; + fi + } + + check "does not configure sccache-dist auth when no GH_TOKEN or SCCACHE_DIST_TOKEN" does_not_set_sccache_dist_auth_config; + + uses_GH_TOKEN_as_sccache_dist_auth_token() { + reset_state; + GH_TOKEN="${gh_token}" \ + DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ + SCCACHE_DIST_SCHEDULER_URL="${sccache_dist_scheduler_url}" \ + devcontainer-utils-post-attach-command; + expect_sccache_dist_auth_token_is_gh_token; + } + + check "configures sccache-dist to use GH_TOKEN as auth token" uses_GH_TOKEN_as_sccache_dist_auth_token; + + uses_SCCACHE_DIST_TOKEN_as_sccache_dist_auth_token() { + reset_state; + SCCACHE_DIST_TOKEN="${gh_token}" \ + DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ + SCCACHE_DIST_SCHEDULER_URL="${sccache_dist_scheduler_url}" \ + devcontainer-utils-post-attach-command; + expect_sccache_dist_auth_token_is_gh_token; + } + + check "configures sccache-dist to use SCCACHE_DIST_TOKEN as auth token" uses_SCCACHE_DIST_TOKEN_as_sccache_dist_auth_token; +fi + # Report result # If any of the checks above exited with a non-zero exit code, the test will fail. reportResults diff --git a/features/test/utils/ubuntu20.04.sh b/features/test/utils/ubuntu20.04.sh index d260ea905..581a4ecd6 100644 --- a/features/test/utils/ubuntu20.04.sh +++ b/features/test/utils/ubuntu20.04.sh @@ -18,6 +18,7 @@ source dev-container-features-test-lib export VAULT_S3_TTL="${VAULT_S3_TTL:-"900"}"; cp ~/.bashrc /tmp/.bashrc-clean; +sudo cp /usr/bin/sccache{,.orig}; utils_profile_script="$(find /etc/profile.d/ -type f -name '*-devcontainer-utils.sh')"; @@ -49,19 +50,24 @@ reset_state() { unset AWS_SESSION_TOKEN; unset AWS_SECRET_ACCESS_KEY; - rm -rf ~/.aws/ ~/.config/gh/; + while pgrep sccache >/dev/null 2>&1; do + devcontainer-utils-stop-sccache --kill-all; + done + + sudo rm -rf \ + ~/.aws/ \ + ~/.config/gh/ \ + ~/.config/sccache/config \ + /var/log/devcontainer-utils/creds-s3.log; cp /tmp/.bashrc-clean ~/.bashrc; + sudo cp /usr/bin/sccache{.orig,}; echo "#! /usr/bin/env bash" | sudo tee "${utils_profile_script}" >/dev/null; sudo chmod +x "${utils_profile_script}"; . ~/.bashrc; - - if test -n "$(pgrep sccache || echo)"; then - sccache --stop-server >/dev/null 2>&1 || true; - fi } write_bad_creds() { - devcontainer-utils-vault-s3-creds-persist <<< " + devcontainer-utils-creds-s3-persist <<< " --stamp='$(date '+%s')' --bucket='${rw_sccache_bucket:-}' --region='${rw_sccache_region:-}' @@ -83,6 +89,14 @@ expect_local_disk_cache_is_used() { grep -qE 'Cache location \s+ Local disk' <<< "${stats}"; } +expect_sccache_dist_auth_token_is_gh_token() { + set -e; + test -f ~/.config/sccache/config; + grep -qE "[dist.auth]" ~/.config/sccache/config; + grep -qE "type = \"token\"" ~/.config/sccache/config; + grep -qE "token = \"${gh_token}\"" ~/.config/sccache/config; +} + if test -n "${vault_host:+x}" \ && test -n "${rw_sccache_bucket:+x}"; then @@ -245,6 +259,44 @@ if test -n "${gh_token:+x}" \ check "bad stored creds with GH_TOKEN, AWS_ROLE_ARN, and SCCACHE_BUCKET should regenerate credentials" bad_stored_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_regenerate_credentials; fi +if test -n "${gh_token:-}" \ +&& test -n "${sccache_dist_scheduler_url:-}"; then + + does_not_set_sccache_dist_auth_config() { + reset_state; + DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ + SCCACHE_DIST_SCHEDULER_URL="${sccache_dist_scheduler_url}" \ + devcontainer-utils-post-attach-command; + if expect_sccache_dist_auth_token_is_gh_token; then + return 1; + fi + } + + check "does not configure sccache-dist auth when no GH_TOKEN or SCCACHE_DIST_TOKEN" does_not_set_sccache_dist_auth_config; + + uses_GH_TOKEN_as_sccache_dist_auth_token() { + reset_state; + GH_TOKEN="${gh_token}" \ + DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ + SCCACHE_DIST_SCHEDULER_URL="${sccache_dist_scheduler_url}" \ + devcontainer-utils-post-attach-command; + expect_sccache_dist_auth_token_is_gh_token; + } + + check "configures sccache-dist to use GH_TOKEN as auth token" uses_GH_TOKEN_as_sccache_dist_auth_token; + + uses_SCCACHE_DIST_TOKEN_as_sccache_dist_auth_token() { + reset_state; + SCCACHE_DIST_TOKEN="${gh_token}" \ + DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ + SCCACHE_DIST_SCHEDULER_URL="${sccache_dist_scheduler_url}" \ + devcontainer-utils-post-attach-command; + expect_sccache_dist_auth_token_is_gh_token; + } + + check "configures sccache-dist to use SCCACHE_DIST_TOKEN as auth token" uses_SCCACHE_DIST_TOKEN_as_sccache_dist_auth_token; +fi + # Report result # If any of the checks above exited with a non-zero exit code, the test will fail. reportResults diff --git a/features/test/utils/ubuntu22.04.sh b/features/test/utils/ubuntu22.04.sh index d260ea905..581a4ecd6 100644 --- a/features/test/utils/ubuntu22.04.sh +++ b/features/test/utils/ubuntu22.04.sh @@ -18,6 +18,7 @@ source dev-container-features-test-lib export VAULT_S3_TTL="${VAULT_S3_TTL:-"900"}"; cp ~/.bashrc /tmp/.bashrc-clean; +sudo cp /usr/bin/sccache{,.orig}; utils_profile_script="$(find /etc/profile.d/ -type f -name '*-devcontainer-utils.sh')"; @@ -49,19 +50,24 @@ reset_state() { unset AWS_SESSION_TOKEN; unset AWS_SECRET_ACCESS_KEY; - rm -rf ~/.aws/ ~/.config/gh/; + while pgrep sccache >/dev/null 2>&1; do + devcontainer-utils-stop-sccache --kill-all; + done + + sudo rm -rf \ + ~/.aws/ \ + ~/.config/gh/ \ + ~/.config/sccache/config \ + /var/log/devcontainer-utils/creds-s3.log; cp /tmp/.bashrc-clean ~/.bashrc; + sudo cp /usr/bin/sccache{.orig,}; echo "#! /usr/bin/env bash" | sudo tee "${utils_profile_script}" >/dev/null; sudo chmod +x "${utils_profile_script}"; . ~/.bashrc; - - if test -n "$(pgrep sccache || echo)"; then - sccache --stop-server >/dev/null 2>&1 || true; - fi } write_bad_creds() { - devcontainer-utils-vault-s3-creds-persist <<< " + devcontainer-utils-creds-s3-persist <<< " --stamp='$(date '+%s')' --bucket='${rw_sccache_bucket:-}' --region='${rw_sccache_region:-}' @@ -83,6 +89,14 @@ expect_local_disk_cache_is_used() { grep -qE 'Cache location \s+ Local disk' <<< "${stats}"; } +expect_sccache_dist_auth_token_is_gh_token() { + set -e; + test -f ~/.config/sccache/config; + grep -qE "[dist.auth]" ~/.config/sccache/config; + grep -qE "type = \"token\"" ~/.config/sccache/config; + grep -qE "token = \"${gh_token}\"" ~/.config/sccache/config; +} + if test -n "${vault_host:+x}" \ && test -n "${rw_sccache_bucket:+x}"; then @@ -245,6 +259,44 @@ if test -n "${gh_token:+x}" \ check "bad stored creds with GH_TOKEN, AWS_ROLE_ARN, and SCCACHE_BUCKET should regenerate credentials" bad_stored_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_regenerate_credentials; fi +if test -n "${gh_token:-}" \ +&& test -n "${sccache_dist_scheduler_url:-}"; then + + does_not_set_sccache_dist_auth_config() { + reset_state; + DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ + SCCACHE_DIST_SCHEDULER_URL="${sccache_dist_scheduler_url}" \ + devcontainer-utils-post-attach-command; + if expect_sccache_dist_auth_token_is_gh_token; then + return 1; + fi + } + + check "does not configure sccache-dist auth when no GH_TOKEN or SCCACHE_DIST_TOKEN" does_not_set_sccache_dist_auth_config; + + uses_GH_TOKEN_as_sccache_dist_auth_token() { + reset_state; + GH_TOKEN="${gh_token}" \ + DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ + SCCACHE_DIST_SCHEDULER_URL="${sccache_dist_scheduler_url}" \ + devcontainer-utils-post-attach-command; + expect_sccache_dist_auth_token_is_gh_token; + } + + check "configures sccache-dist to use GH_TOKEN as auth token" uses_GH_TOKEN_as_sccache_dist_auth_token; + + uses_SCCACHE_DIST_TOKEN_as_sccache_dist_auth_token() { + reset_state; + SCCACHE_DIST_TOKEN="${gh_token}" \ + DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ + SCCACHE_DIST_SCHEDULER_URL="${sccache_dist_scheduler_url}" \ + devcontainer-utils-post-attach-command; + expect_sccache_dist_auth_token_is_gh_token; + } + + check "configures sccache-dist to use SCCACHE_DIST_TOKEN as auth token" uses_SCCACHE_DIST_TOKEN_as_sccache_dist_auth_token; +fi + # Report result # If any of the checks above exited with a non-zero exit code, the test will fail. reportResults From 3932245bbb603bdf7ab6626ba38477456566d5f2 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 16 Apr 2025 14:37:23 -0700 Subject: [PATCH 02/78] remove -DCMAKE_POLICY_VERSION_MINIMUM=3.5 --- .github/workflows/build-all-rapids-repos.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index f531fa60e..20e33f990 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -92,8 +92,7 @@ jobs: -DBUILD_PRIMS_BENCH=ON \ -DBUILD_SHARED_LIBS=ON \ -DRAFT_COMPILE_LIBRARY=ON \ - -DBUILD_CUGRAPH_MG_TESTS=ON \ - -DCMAKE_POLICY_VERSION_MINIMUM=3.5; + -DBUILD_CUGRAPH_MG_TESTS=ON ; # Build all the libs build-all -j100000; From 1de075e93d70c5eedf8c4109e5c5381fd902a2be Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 16 Apr 2025 15:05:36 -0700 Subject: [PATCH 03/78] temporarily use my cuspatial fork --- .../src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml index 13193db11..4be07da04 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml @@ -232,7 +232,7 @@ repos: - name: cuspatial path: cuspatial - git: {<<: *git_defaults, repo: cuspatial} + git: {<<: *git_defaults, repo: cuspatial, upstream: trxcllnt, tag: fix/vendor-rapids-cmake} cpp: - name: cuspatial sub_dir: cpp From 36ad779245541dabdb18d2e06810b03336716442 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 17 Apr 2025 08:07:33 -0700 Subject: [PATCH 04/78] remove cub and thrust namespace mangling flags --- .github/workflows/build-all-rapids-repos.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 20e33f990..6e9701acc 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -53,14 +53,6 @@ jobs: # Never fallback to locally compiling export SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false - # Disable CUB/Thrust arch-dependent namespaces. - # These prevent reusing ptx and cubins from multi-arch - # compilations in single-arch subset compilations. - for VAR in CFLAGS CXXFLAGS CUDAFLAGS; do - export "$VAR=${!VAR:+${!VAR} }-DCUB_DISABLE_NAMESPACE_MAGIC -DCUB_IGNORE_NAMESPACE_MAGIC_ERROR" - export "$VAR=${!VAR:+${!VAR} }-DTHRUST_DISABLE_ABI_NAMESPACE -DTHRUST_IGNORE_ABI_NAMESPACE_ERROR" - done - # Install the latest sccache client devcontainer-utils-install-sccache --repo trxcllnt/sccache; From 8b60e23e6c01453948e47f49fe58f3e37fd89579 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 17 Apr 2025 08:54:08 -0700 Subject: [PATCH 05/78] cpu8 -> cpu4 --- .github/workflows/build-all-rapids-repos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 6e9701acc..5def6e910 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -37,7 +37,7 @@ jobs: with: arch: '["amd64", "arm64"]' cuda: '["12.8"]' - node_type: cpu8 + node_type: cpu4 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN build_command: | From e07b0ddd7b85dfe7f5ba55bd8dbd1cec99f54606 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 17 Apr 2025 09:02:52 -0700 Subject: [PATCH 06/78] test shared-workflows fea/devcontainers-job-timeout changes --- .github/workflows/build-all-rapids-repos.yml | 24 +++++++++----------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 5def6e910..bca8d5f17 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -27,7 +27,7 @@ jobs: if: needs.check-event.outputs.ok == 'true' needs: check-event secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.06 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fea/devcontainers-job-timeout permissions: actions: read packages: read @@ -40,19 +40,17 @@ jobs: node_type: cpu4 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN + timeout-minutes: 720 + # 1. Repopulate the cache + # 2. Prevent the sccache server from shutting down + # 3. Infinitely retry transient errors + # 4. Never fallback to locally compiling + env: | + SCCACHE_RECACHE=1 + SCCACHE_IDLE_TIMEOUT=0 + SCCACHE_DIST_MAX_RETRIES=inf + SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false build_command: | - # Repopulate the cache - export SCCACHE_RECACHE=1 - - # Prevent the sccache server from shutting down - export SCCACHE_IDLE_TIMEOUT=0 - - # Infinitely retry transient errors - export SCCACHE_DIST_MAX_RETRIES=inf - - # Never fallback to locally compiling - export SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false - # Install the latest sccache client devcontainer-utils-install-sccache --repo trxcllnt/sccache; From 5c138aa719c73642e0abbd7dbb6813d379a56153 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 17 Apr 2025 16:11:10 -0700 Subject: [PATCH 07/78] rename SCCACHE_DIST_SCHEDULER_URL to SCCACHE_DIST_URL --- .devcontainer/rapids.Dockerfile | 2 +- .../devcontainer/bin/post-attach-command.sh | 17 +- .../bin/sccache/dist/configure.sh | 146 +++++++++++------- .../opt/devcontainer/bin/sccache/dist/init.sh | 30 ++-- .../devcontainer/bin/sccache/dist/status.sh | 2 +- features/test/utils/ubuntu18.04.sh | 27 ++-- features/test/utils/ubuntu20.04.sh | 27 ++-- features/test/utils/ubuntu22.04.sh | 27 ++-- 8 files changed, 152 insertions(+), 126 deletions(-) diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile index ae5a8b8ac..728ccdb7c 100644 --- a/.devcontainer/rapids.Dockerfile +++ b/.devcontainer/rapids.Dockerfile @@ -53,7 +53,7 @@ ENV SCCACHE_REGION="us-east-2" ENV SCCACHE_BUCKET="rapids-sccache-devs" ENV SCCACHE_DIST_CONNECT_TIMEOUT=30 ENV SCCACHE_DIST_REQUEST_TIMEOUT=1800 -ENV SCCACHE_DIST_SCHEDULER_URL="https://${TARGETARCH}.linux.sccache.gha-runners.nvidia.com" +ENV SCCACHE_DIST_URL="https://${TARGETARCH}.linux.sccache.gha-runners.nvidia.com" ENV SCCACHE_IDLE_TIMEOUT=1800 ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs" diff --git a/features/src/utils/opt/devcontainer/bin/post-attach-command.sh b/features/src/utils/opt/devcontainer/bin/post-attach-command.sh index 700b99350..1002a0399 100755 --- a/features/src/utils/opt/devcontainer/bin/post-attach-command.sh +++ b/features/src/utils/opt/devcontainer/bin/post-attach-command.sh @@ -9,26 +9,21 @@ if ! test -n "${SKIP_DEVCONTAINER_UTILS_POST_ATTACH_COMMAND:+x}"; then . devcontainer-utils-init-git-interactive; # Reset sccache-dist configuration - devcontainer-utils-init-sccache-dist; + SCCACHE_DIST_URL= devcontainer-utils-configure-sccache-dist; # Initialize the GitHub CLI with the appropriate user scopes + # and load credentials for the sccache client # shellcheck disable=SC1091 . devcontainer-utils-creds-s3-init; # Update sccache client configuration to enable/disable sccache-dist if test -n "${DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST:+x}"; then - if test -n "${SCCACHE_DIST_TOKEN:+x}"; then - devcontainer-utils-install-sccache --repo trxcllnt/sccache; - # Update ~/.config/sccache/config to use $SCCACHE_DIST_TOKEN - devcontainer-utils-init-sccache-dist \ - --enable-sccache-dist - <<< " \ - --auth-type 'token' \ - --auth-token '${SCCACHE_DIST_TOKEN}' \ - "; - elif gh auth status >/dev/null 2>&1; then - devcontainer-utils-install-sccache --repo trxcllnt/sccache; + devcontainer-utils-install-sccache --repo trxcllnt/sccache; + if ! test -n "${SCCACHE_DIST_AUTH_TOKEN:+x}" && gh auth status >/dev/null 2>&1; then # Update ~/.config/sccache/config to use gh token auth devcontainer-utils-init-sccache-dist --enable-with-github-auth; + else + devcontainer-utils-init-sccache-dist; fi fi fi diff --git a/features/src/utils/opt/devcontainer/bin/sccache/dist/configure.sh b/features/src/utils/opt/devcontainer/bin/sccache/dist/configure.sh index 319d87857..3c4e5ef7b 100755 --- a/features/src/utils/opt/devcontainer/bin/sccache/dist/configure.sh +++ b/features/src/utils/opt/devcontainer/bin/sccache/dist/configure.sh @@ -9,8 +9,9 @@ # -h,--help Print this text. # --no-local-compile-fallback Disable building locally after retrying transient sccache-dist errors. # (default: false) -# --use-connection-pool Enable sccache client HTTP connection pool. +# --connection-pool Enable sccache client HTTP connection pool. # (default: false) +# --keepalive Enable sccache client HTTP keepalive. # # Options that require values: # --auth-type (token|oauth2_code_grant_pkce|oauth2_implicit) Auth type for build cluster auth. @@ -20,9 +21,13 @@ # --auth-url Auth URL used to authenticate with the build cluster when `--auth-type=(oauth2_code_grant_pkce|oauth2_implicit)`. # --token-url Token URL used to authenticate with the build cluster when `--auth-type=oauth2_code_grant_pkce`. # --connect-timeout The sccache client HTTP connection timeout. -# (default: 30) +# (default: 5) # --request-timeout The sccache client HTTP request timeout. -# (default: 1800) +# (default: 600) +# --keepalive-interval The sccache HTTP keepalive interval +# (default: 20) +# --keepalive-timeout The sccache HTTP keepalive timeout +# (default: 60) # --scheduler-url URL of the sccache-dist build cluster. # --max-retries Maximum number of times to retry transient sccache-dist errors. # Pass `--max-retries inf` to retry infinitely. @@ -30,6 +35,9 @@ # ensures the sccache client relies exclusively on the build cluster. # (default: 0) +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/../../update-envvars.sh"; + _configure_sccache_dist() { local -; set -euo pipefail; @@ -39,24 +47,6 @@ _configure_sccache_dist() { # shellcheck disable=SC1091 . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'sccache configure-sccache-dist'; - auth_type="${auth_type-}"; - max_retries="${max_retries:-${SCCACHE_DIST_MAX_RETRIES-}}"; - scheduler_url="${scheduler_url:-${SCCACHE_DIST_SCHEDULER_URL-}}"; - connect_timeout="${connect_timeout:-${SCCACHE_DIST_CONNECT_TIMEOUT-}}"; - request_timeout="${request_timeout:-${SCCACHE_DIST_REQUEST_TIMEOUT-}}"; - - local connection_pool="${SCCACHE_DIST_CONNECTION_POOL-}"; - - if test -n "${use_connection_pool:+x}"; then - connection_pool="true"; - fi - - local fallback_to_local_compile="${SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE-}"; - - if test -n "${no_local_compile_fallback:+x}"; then - fallback_to_local_compile="false"; - fi - local sccache_conf="${SCCACHE_CONF:-"$HOME/.config/sccache/config"}"; mkdir -p "$(dirname "$sccache_conf")"; @@ -70,61 +60,101 @@ _configure_sccache_dist() { # # [dist.net] # connection_pool = true - # connect_timeout = 10 + # connect_timeout = 5 # request_timeout = 600 # + # [dist.net.keepalive] + # enabled = true + # interval = 20 + # timeout = 60 + # # [dist.auth] # type = "foo" # token = "bar" - sed -Ei '/^\[dist.*\]/,/^(\s|\[)*$/d' "${sccache_conf}"; + + while grep -qE "^\[dist.*\]" "${sccache_conf}"; do + sed -Ei '/^\[dist.*\]/,/^(\s|\[)*$/d' "${sccache_conf}"; + done + + printf "%s\n" "$(< "${sccache_conf}")" > "${sccache_conf}.new" + mv "${sccache_conf}"{.new,}; + + scheduler_url="${scheduler_url:-${SCCACHE_DIST_URL-}}"; # Write our new values - if ! grep -qE "^$" <<< "${scheduler_url}"; then + if test -n "${scheduler_url:+x}"; then + cat <<< "" >> "${sccache_conf}"; cat <<< "[dist]" >> "${sccache_conf}"; cat <<< "scheduler_url = \"${scheduler_url}\"" >> "${sccache_conf}"; - if ! grep -qE "^$" <<< "${max_retries}"; then + if test -n "${max_retries:+x}"; then cat <<< "max_retries = ${max_retries}" >> "${sccache_conf}"; fi - if ! grep -qE "^$" <<< "${fallback_to_local_compile}"; then - cat <<< "fallback_to_local_compile = ${fallback_to_local_compile}" >> "${sccache_conf}"; + if test -n "${no_local_compile_fallback:+x}"; then + cat <<< "fallback_to_local_compile = false" >> "${sccache_conf}"; fi - cat <<< "[dist.net]" >> "${sccache_conf}"; - if ! grep -qE "^$" <<< "${connection_pool}"; then - cat <<< "connection_pool = ${connection_pool}" >> "${sccache_conf}"; - fi - if ! grep -qE "^$" <<< "${connect_timeout}"; then - cat <<< "connect_timeout = "${connect_timeout}"" >> "${sccache_conf}"; - fi - if ! grep -qE "^$" <<< "${request_timeout}"; then - cat <<< "request_timeout = "${request_timeout}"" >> "${sccache_conf}"; + if test -n "${connection_pool:+x}" \ + || test -n "${connect_timeout:+x}" \ + || test -n "${request_timeout:+x}"; then + cat <<< "" >> "${sccache_conf}"; + cat <<< "[dist.net]" >> "${sccache_conf}"; + if test -n "${connection_pool:+x}"; then + cat <<< "connection_pool = true" >> "${sccache_conf}"; + fi + if test -n "${connect_timeout:+x}"; then + cat <<< "connect_timeout = "${connect_timeout}"" >> "${sccache_conf}"; + fi + if test -n "${request_timeout:+x}"; then + cat <<< "request_timeout = "${request_timeout}"" >> "${sccache_conf}"; + fi fi - if test "${auth_type}" = token; then - if ! grep -qE "^$" <<< "${auth_token-}"; then - cat <<< "[dist.auth]" >> "${sccache_conf}"; - cat <<< "type = \"${auth_type}\"" >> "${sccache_conf}"; - cat <<< "token = \"${auth_token}\"" >> "${sccache_conf}"; + if test -n "${keepalive:+x}"; then + cat <<< "" >> "${sccache_conf}"; + cat <<< "[dist.net.keepalive]" >> "${sccache_conf}"; + cat <<< "enabled = true" >> "${sccache_conf}"; + if test -n "${keepalive_interval:+x}"; then + cat <<< "interval = "${keepalive_interval}"" >> "${sccache_conf}"; fi - elif test "${auth_type}" = oauth2_implicit; then - if ! grep -qE "^$" <<< "${auth_url-}" \ - && ! grep -qE "^$" <<< "${client_id-}"; then - cat <<< "[dist.auth]" >> "${sccache_conf}"; - cat <<< "type = \"${auth_type}\"" >> "${sccache_conf}"; - cat <<< "auth_url = \"${auth_url}\"" >> "${sccache_conf}"; - cat <<< "client_id = \"${client_id}\"" >> "${sccache_conf}"; - fi - elif test "${auth_type}" = oauth2_code_grant_pkce; then - if ! grep -qE "^$" <<< "${auth_url-}" \ - && ! grep -qE "^$" <<< "${client_id-}" \ - && ! grep -qE "^$" <<< "${token_url-}"; then - cat <<< "[dist.auth]" >> "${sccache_conf}"; - cat <<< "type = \"${auth_type}\"" >> "${sccache_conf}"; - cat <<< "auth_url = \"${auth_url}\"" >> "${sccache_conf}"; - cat <<< "client_id = \"${client_id}\"" >> "${sccache_conf}"; - cat <<< "token_url = \"${token_url}\"" >> "${sccache_conf}"; + if test -n "${keepalive_timeout:+x}"; then + cat <<< "timeout = "${keepalive_timeout}"" >> "${sccache_conf}"; fi fi + + case "${auth_type:-}" in + token) + if test -n "${auth_token:+x}"; then + cat <<< "" >> "${sccache_conf}"; + cat <<< "[dist.auth]" >> "${sccache_conf}"; + cat <<< "type = \"${auth_type}\"" >> "${sccache_conf}"; + cat <<< "token = \"${auth_token}\"" >> "${sccache_conf}"; + fi + ;; + oauth2_implicit) + if test -n "${auth_url:+x}" \ + && test -n "${client_id:+x}"; then + cat <<< "" >> "${sccache_conf}"; + cat <<< "[dist.auth]" >> "${sccache_conf}"; + cat <<< "type = \"${auth_type}\"" >> "${sccache_conf}"; + cat <<< "auth_url = \"${auth_url}\"" >> "${sccache_conf}"; + cat <<< "client_id = \"${client_id}\"" >> "${sccache_conf}"; + fi + ;; + oauth2_code_grant_pkce) + if test -n "${auth_url:+x}" \ + && test -n "${client_id:+x}" \ + && test -n "${token_url:+x}"; then + cat <<< "" >> "${sccache_conf}"; + cat <<< "[dist.auth]" >> "${sccache_conf}"; + cat <<< "type = \"${auth_type}\"" >> "${sccache_conf}"; + cat <<< "auth_url = \"${auth_url}\"" >> "${sccache_conf}"; + cat <<< "client_id = \"${client_id}\"" >> "${sccache_conf}"; + cat <<< "token_url = \"${token_url}\"" >> "${sccache_conf}"; + fi + ;; + *) + ;; + esac fi } diff --git a/features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh b/features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh index 624656407..e5a21d36b 100755 --- a/features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh +++ b/features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh @@ -15,8 +15,9 @@ # (default: false) # --no-local-compile-fallback Disable building locally after retrying transient sccache-dist errors. # (default: false) -# --use-connection-pool Enable sccache client HTTP connection pool. +# --connection-pool Enable sccache client HTTP connection pool. # (default: false) +# --keepalive Enable sccache client HTTP keepalive. # # Options that require values: # --auth-type (token|oauth2_code_grant_pkce|oauth2_implicit) Auth type for build cluster auth. @@ -26,9 +27,13 @@ # --auth-url Auth URL used to authenticate with the build cluster when `--auth-type=(oauth2_code_grant_pkce|oauth2_implicit)`. # --token-url Token URL used to authenticate with the build cluster when `--auth-type=oauth2_code_grant_pkce`. # --connect-timeout The sccache client HTTP connection timeout. -# (default: 30) +# (default: 5) # --request-timeout The sccache client HTTP request timeout. -# (default: 1800) +# (default: 600) +# --keepalive-interval The sccache HTTP keepalive interval +# (default: 20) +# --keepalive-timeout The sccache HTTP keepalive timeout +# (default: 60) # --scheduler-url URL of the sccache-dist build cluster. # --max-retries Maximum number of times to retry transient sccache-dist errors. # Pass `--max-retries inf` to retry infinitely. @@ -40,26 +45,19 @@ _init_sccache_dist() { local -; set -euo pipefail; - eval "$(devcontainer-utils-parse-args "$0" --take '-e,--enable-sccache-dist --enable-with-github-auth' "$@" <<< "")"; + eval "$(devcontainer-utils-parse-args "$0" --take '-e,--enable-sccache-dist --enable-with-github-auth' "$@" <&0)"; # shellcheck disable=SC1091 . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'sccache init-sccache-dist'; if test -n "${enable_sccache_dist:+x}"; then - # Add sccache-dist configuration to ~/.config/sccache/config - devcontainer-utils-configure-sccache-dist "${OPTS[@]}" - <&0; + # Add sccache-dist configuration + devcontainer-utils-configure-sccache-dist - <<< "${OPTS[*]@Q}"; elif test -n "${enable_with_github_auth:+x}"; then - devcontainer-utils-configure-sccache-dist - <<< " \ - --auth-type 'token' --auth-token '$(gh auth token)'"; + devcontainer-utils-configure-sccache-dist - <<< "--auth-type token --auth-token '$(gh auth token)' ${OPTS[*]@Q}"; else - # Delete sccache-dist configuration from ~/.config/sccache/config - SCCACHE_DIST_MAX_RETRIES= \ - SCCACHE_DIST_SCHEDULER_URL= \ - SCCACHE_DIST_CONNECT_TIMEOUT= \ - SCCACHE_DIST_REQUEST_TIMEOUT= \ - SCCACHE_DIST_CONNECTION_POOL= \ - SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE= \ - devcontainer-utils-configure-sccache-dist; + # Reset sccache-dist configuration + SCCACHE_DIST_URL= devcontainer-utils-configure-sccache-dist; fi # Restart the sccache client with the new configuration diff --git a/features/src/utils/opt/devcontainer/bin/sccache/dist/status.sh b/features/src/utils/opt/devcontainer/bin/sccache/dist/status.sh index dead1e8ff..61749ee4d 100755 --- a/features/src/utils/opt/devcontainer/bin/sccache/dist/status.sh +++ b/features/src/utils/opt/devcontainer/bin/sccache/dist/status.sh @@ -52,7 +52,7 @@ _sccache_dist_status() { id: .id, servers: (if .servers == null then "-" else (.servers | length) end), cpus: .info.occupancy, - util: ((.info.cpu_usage // 0) * 100 | round | . / 100 | tostring | . + "%"), + util: ((.info.cpu_usage // 0) * 1000 | floor | . / 1000 | tostring | . + "%"), jobs: (.jobs.loading + .jobs.pending + .jobs.running), loading: .jobs.loading, pending: .jobs.pending, diff --git a/features/test/utils/ubuntu18.04.sh b/features/test/utils/ubuntu18.04.sh index 581a4ecd6..866cad28a 100644 --- a/features/test/utils/ubuntu18.04.sh +++ b/features/test/utils/ubuntu18.04.sh @@ -265,36 +265,37 @@ if test -n "${gh_token:-}" \ does_not_set_sccache_dist_auth_config() { reset_state; DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ - SCCACHE_DIST_SCHEDULER_URL="${sccache_dist_scheduler_url}" \ + SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ devcontainer-utils-post-attach-command; if expect_sccache_dist_auth_token_is_gh_token; then return 1; fi } - check "does not configure sccache-dist auth when no GH_TOKEN or SCCACHE_DIST_TOKEN" does_not_set_sccache_dist_auth_config; + check "does not configure sccache-dist auth when no GH_TOKEN or SCCACHE_DIST_AUTH_TOKEN" does_not_set_sccache_dist_auth_config; uses_GH_TOKEN_as_sccache_dist_auth_token() { reset_state; GH_TOKEN="${gh_token}" \ DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ - SCCACHE_DIST_SCHEDULER_URL="${sccache_dist_scheduler_url}" \ + SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ devcontainer-utils-post-attach-command; expect_sccache_dist_auth_token_is_gh_token; } check "configures sccache-dist to use GH_TOKEN as auth token" uses_GH_TOKEN_as_sccache_dist_auth_token; - uses_SCCACHE_DIST_TOKEN_as_sccache_dist_auth_token() { - reset_state; - SCCACHE_DIST_TOKEN="${gh_token}" \ - DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ - SCCACHE_DIST_SCHEDULER_URL="${sccache_dist_scheduler_url}" \ - devcontainer-utils-post-attach-command; - expect_sccache_dist_auth_token_is_gh_token; - } - - check "configures sccache-dist to use SCCACHE_DIST_TOKEN as auth token" uses_SCCACHE_DIST_TOKEN_as_sccache_dist_auth_token; + # uses_SCCACHE_DIST_AUTH_TOKEN_as_sccache_dist_auth_token() { + # reset_state; + # SCCACHE_DIST_AUTH_TYPE=token \ + # SCCACHE_DIST_AUTH_TOKEN="${gh_token}" \ + # DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ + # SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ + # devcontainer-utils-post-attach-command; + # expect_sccache_dist_auth_token_is_gh_token; + # } + + # check "configures sccache-dist to use SCCACHE_DIST_AUTH_TOKEN as auth token" uses_SCCACHE_DIST_AUTH_TOKEN_as_sccache_dist_auth_token; fi # Report result diff --git a/features/test/utils/ubuntu20.04.sh b/features/test/utils/ubuntu20.04.sh index 581a4ecd6..866cad28a 100644 --- a/features/test/utils/ubuntu20.04.sh +++ b/features/test/utils/ubuntu20.04.sh @@ -265,36 +265,37 @@ if test -n "${gh_token:-}" \ does_not_set_sccache_dist_auth_config() { reset_state; DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ - SCCACHE_DIST_SCHEDULER_URL="${sccache_dist_scheduler_url}" \ + SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ devcontainer-utils-post-attach-command; if expect_sccache_dist_auth_token_is_gh_token; then return 1; fi } - check "does not configure sccache-dist auth when no GH_TOKEN or SCCACHE_DIST_TOKEN" does_not_set_sccache_dist_auth_config; + check "does not configure sccache-dist auth when no GH_TOKEN or SCCACHE_DIST_AUTH_TOKEN" does_not_set_sccache_dist_auth_config; uses_GH_TOKEN_as_sccache_dist_auth_token() { reset_state; GH_TOKEN="${gh_token}" \ DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ - SCCACHE_DIST_SCHEDULER_URL="${sccache_dist_scheduler_url}" \ + SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ devcontainer-utils-post-attach-command; expect_sccache_dist_auth_token_is_gh_token; } check "configures sccache-dist to use GH_TOKEN as auth token" uses_GH_TOKEN_as_sccache_dist_auth_token; - uses_SCCACHE_DIST_TOKEN_as_sccache_dist_auth_token() { - reset_state; - SCCACHE_DIST_TOKEN="${gh_token}" \ - DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ - SCCACHE_DIST_SCHEDULER_URL="${sccache_dist_scheduler_url}" \ - devcontainer-utils-post-attach-command; - expect_sccache_dist_auth_token_is_gh_token; - } - - check "configures sccache-dist to use SCCACHE_DIST_TOKEN as auth token" uses_SCCACHE_DIST_TOKEN_as_sccache_dist_auth_token; + # uses_SCCACHE_DIST_AUTH_TOKEN_as_sccache_dist_auth_token() { + # reset_state; + # SCCACHE_DIST_AUTH_TYPE=token \ + # SCCACHE_DIST_AUTH_TOKEN="${gh_token}" \ + # DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ + # SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ + # devcontainer-utils-post-attach-command; + # expect_sccache_dist_auth_token_is_gh_token; + # } + + # check "configures sccache-dist to use SCCACHE_DIST_AUTH_TOKEN as auth token" uses_SCCACHE_DIST_AUTH_TOKEN_as_sccache_dist_auth_token; fi # Report result diff --git a/features/test/utils/ubuntu22.04.sh b/features/test/utils/ubuntu22.04.sh index 581a4ecd6..866cad28a 100644 --- a/features/test/utils/ubuntu22.04.sh +++ b/features/test/utils/ubuntu22.04.sh @@ -265,36 +265,37 @@ if test -n "${gh_token:-}" \ does_not_set_sccache_dist_auth_config() { reset_state; DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ - SCCACHE_DIST_SCHEDULER_URL="${sccache_dist_scheduler_url}" \ + SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ devcontainer-utils-post-attach-command; if expect_sccache_dist_auth_token_is_gh_token; then return 1; fi } - check "does not configure sccache-dist auth when no GH_TOKEN or SCCACHE_DIST_TOKEN" does_not_set_sccache_dist_auth_config; + check "does not configure sccache-dist auth when no GH_TOKEN or SCCACHE_DIST_AUTH_TOKEN" does_not_set_sccache_dist_auth_config; uses_GH_TOKEN_as_sccache_dist_auth_token() { reset_state; GH_TOKEN="${gh_token}" \ DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ - SCCACHE_DIST_SCHEDULER_URL="${sccache_dist_scheduler_url}" \ + SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ devcontainer-utils-post-attach-command; expect_sccache_dist_auth_token_is_gh_token; } check "configures sccache-dist to use GH_TOKEN as auth token" uses_GH_TOKEN_as_sccache_dist_auth_token; - uses_SCCACHE_DIST_TOKEN_as_sccache_dist_auth_token() { - reset_state; - SCCACHE_DIST_TOKEN="${gh_token}" \ - DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ - SCCACHE_DIST_SCHEDULER_URL="${sccache_dist_scheduler_url}" \ - devcontainer-utils-post-attach-command; - expect_sccache_dist_auth_token_is_gh_token; - } - - check "configures sccache-dist to use SCCACHE_DIST_TOKEN as auth token" uses_SCCACHE_DIST_TOKEN_as_sccache_dist_auth_token; + # uses_SCCACHE_DIST_AUTH_TOKEN_as_sccache_dist_auth_token() { + # reset_state; + # SCCACHE_DIST_AUTH_TYPE=token \ + # SCCACHE_DIST_AUTH_TOKEN="${gh_token}" \ + # DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ + # SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ + # devcontainer-utils-post-attach-command; + # expect_sccache_dist_auth_token_is_gh_token; + # } + + # check "configures sccache-dist to use SCCACHE_DIST_AUTH_TOKEN as auth token" uses_SCCACHE_DIST_AUTH_TOKEN_as_sccache_dist_auth_token; fi # Report result From ace1e278402d9b97c1766be7afe98cd2c5b03fc9 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 17 Apr 2025 16:11:20 -0700 Subject: [PATCH 08/78] use cpu8 again, clone with --depth 1 --- .github/workflows/build-all-rapids-repos.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index bca8d5f17..dd3f00326 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -37,7 +37,7 @@ jobs: with: arch: '["amd64", "arm64"]' cuda: '["12.8"]' - node_type: cpu4 + node_type: cpu8 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN timeout-minutes: 720 @@ -55,17 +55,17 @@ jobs: devcontainer-utils-install-sccache --repo trxcllnt/sccache; # Configure the sccache client to talk to the build cluster - devcontainer-utils-init-sccache-dist \ - --enable-sccache-dist \ - - <<< "--auth-token '$RAPIDS_AUX_SECRET_1' \ - --auth-type 'token' \ + devcontainer-utils-init-sccache-dist \ + --enable-sccache-dist - <<< " \ + --auth-type 'token' \ + --auth-token '$RAPIDS_AUX_SECRET_1' \ "; # Verify sccache cache location sccache --show-adv-stats; # Clone all the repos - clone-all -j$(nproc) -v -q --clone-upstream --single-branch --shallow-submodules --no-update-env; + clone-all -j$(nproc) -v -q --clone-upstream --depth 1 --single-branch --shallow-submodules --no-update-env; # Create the python env without ninja. # ninja -j10000 fails with `ninja: FATAL: pipe: Too many open files`. @@ -73,9 +73,11 @@ jobs: # so that fix needs to be integrated into the kitware pip ninja builds. rapids-post-start-command --exclude <(echo ninja); + set -x; + # Configure all the C++ libs configure-all \ - -j100000 \ + -j$(ulimit -Hn) \ -Wno-dev \ -DBUILD_TESTS=ON \ -DBUILD_BENCHMARKS=ON \ @@ -85,7 +87,7 @@ jobs: -DBUILD_CUGRAPH_MG_TESTS=ON ; # Build all the libs - build-all -j100000; + build-all -j$(ulimit -Hn); # Print cache and dist stats sccache --show-adv-stats; From 5978e0df38e89e4ee1af61b65e147daac09d6598 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 21 Apr 2025 15:48:08 -0700 Subject: [PATCH 09/78] cleanup --- .../src/utils/opt/devcontainer/bin/sccache/dist/configure.sh | 3 --- features/src/utils/opt/devcontainer/bin/sccache/dist/status.sh | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/features/src/utils/opt/devcontainer/bin/sccache/dist/configure.sh b/features/src/utils/opt/devcontainer/bin/sccache/dist/configure.sh index 3c4e5ef7b..bef92f2f0 100755 --- a/features/src/utils/opt/devcontainer/bin/sccache/dist/configure.sh +++ b/features/src/utils/opt/devcontainer/bin/sccache/dist/configure.sh @@ -35,9 +35,6 @@ # ensures the sccache client relies exclusively on the build cluster. # (default: 0) -# shellcheck disable=SC1091 -. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/../../update-envvars.sh"; - _configure_sccache_dist() { local -; set -euo pipefail; diff --git a/features/src/utils/opt/devcontainer/bin/sccache/dist/status.sh b/features/src/utils/opt/devcontainer/bin/sccache/dist/status.sh index 61749ee4d..050a78f08 100755 --- a/features/src/utils/opt/devcontainer/bin/sccache/dist/status.sh +++ b/features/src/utils/opt/devcontainer/bin/sccache/dist/status.sh @@ -52,7 +52,7 @@ _sccache_dist_status() { id: .id, servers: (if .servers == null then "-" else (.servers | length) end), cpus: .info.occupancy, - util: ((.info.cpu_usage // 0) * 1000 | floor | . / 1000 | tostring | . + "%"), + util: ((.info.cpu_usage // 0) * 100 | floor | . / 100 | tostring | . + "%"), jobs: (.jobs.loading + .jobs.pending + .jobs.running), loading: .jobs.loading, pending: .jobs.pending, From cc2a0324783777d4ef1e83cb1d06abf075fc4227 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 22 Apr 2025 16:40:53 -0700 Subject: [PATCH 10/78] remove -l from ninja args and bump feature versions --- features/src/rapids-build-utils/devcontainer-feature.json | 2 +- .../opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh | 4 +--- .../rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh | 4 ---- .../opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh | 4 ---- features/src/utils/devcontainer-feature.json | 2 +- 5 files changed, 3 insertions(+), 13 deletions(-) diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json index d8df711ed..cad407c26 100644 --- a/features/src/rapids-build-utils/devcontainer-feature.json +++ b/features/src/rapids-build-utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVIDIA RAPIDS devcontainer build utilities", "id": "rapids-build-utils", - "version": "25.6.3", + "version": "25.6.4", "description": "A feature to install the RAPIDS devcontainer build utilities", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh index 4c79bffab..3185f1d88 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh @@ -49,9 +49,7 @@ build_${CPP_LIB}_cpp() { local -a cmake_build_args="($(rapids-select-cmake-build-args ${n_jobs:+-j${n_jobs}} "${OPTS[@]}"))"; cmake \ --build "${CPP_SRC}/${BIN_DIR}" \ - "${cmake_build_args[@]}" \ - -- \ - ${n_load:+-l${n_load}} ; + "${cmake_build_args[@]}" ; { set +x; } 2>/dev/null; echo -n "lib${CPP_LIB} build time:"; ) 2>&1; } diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh index cf99dea91..e4dccbe3d 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh @@ -62,10 +62,6 @@ build_${PY_LIB}_python_wheel() { ninja_args+=("-j${n_jobs}"); fi - if test -n "${n_load:+x}"; then - ninja_args+=("-l${n_load}"); - fi - local -a pip_args="( ${pip_args_+"${pip_args_[*]@Q}"} $(rapids-select-pip-wheel-args "$@") diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh index 9af3fc826..d1457c5bc 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh @@ -60,10 +60,6 @@ install_${PY_LIB}_python() { ninja_args+=("-j${n_jobs}"); fi - if test -n "${n_load:+x}"; then - ninja_args+=("-l${n_load}"); - fi - local -a pip_args="( ${pip_args_+"${pip_args_[*]@Q}"} $(rapids-select-pip-install-args "$@") diff --git a/features/src/utils/devcontainer-feature.json b/features/src/utils/devcontainer-feature.json index ee6251c28..71b0bb213 100644 --- a/features/src/utils/devcontainer-feature.json +++ b/features/src/utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "devcontainer-utils", "id": "utils", - "version": "25.6.3", + "version": "25.6.4", "description": "A feature to install RAPIDS devcontainer utility scripts", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" From 31436cf2dbf94fb65febe562b2d585cbbafee31e Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 30 Apr 2025 10:38:52 -0700 Subject: [PATCH 11/78] switch back to shared-workflows@branch-25.06 --- .github/workflows/build-all-rapids-repos.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index dd3f00326..7743c200e 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -27,7 +27,7 @@ jobs: if: needs.check-event.outputs.ok == 'true' needs: check-event secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fea/devcontainers-job-timeout + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.06 permissions: actions: read packages: read @@ -41,12 +41,10 @@ jobs: extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN timeout-minutes: 720 - # 1. Repopulate the cache - # 2. Prevent the sccache server from shutting down - # 3. Infinitely retry transient errors - # 4. Never fallback to locally compiling + # 1. Prevent the sccache server from shutting down + # 2. Infinitely retry transient errors + # 3. Never fallback to locally compiling env: | - SCCACHE_RECACHE=1 SCCACHE_IDLE_TIMEOUT=0 SCCACHE_DIST_MAX_RETRIES=inf SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false From b126fd123f5f44cb7ea4547335fb8fb18673776d Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 1 May 2025 12:59:19 -0700 Subject: [PATCH 12/78] revert cuspatial chnage in manifest.yaml --- .../src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml index 2b4ade03c..1bc32a90f 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml @@ -236,7 +236,7 @@ repos: - name: cuspatial path: cuspatial - git: {<<: *git_defaults, repo: cuspatial, upstream: trxcllnt, tag: fix/vendor-rapids-cmake} + git: {<<: *git_defaults, repo: cuspatial} cpp: - name: cuspatial sub_dir: cpp From 3cf4f74b3cf0ef90fb273202699ae46d242af048 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 1 May 2025 14:54:04 -0700 Subject: [PATCH 13/78] pass -q option to `git branch --remotes -d` --- .../opt/rapids-build-utils/bin/tmpl/repo.clone.tmpl.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/repo.clone.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/repo.clone.tmpl.sh index 0d1bdbc6a..ae7a5daaa 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/repo.clone.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/repo.clone.tmpl.sh @@ -28,6 +28,7 @@ clone_${NAME}() { --no-update-env -b,--branch -d,--directory + -q,--quiet -u,--upstream --ssh-url --https-url @@ -53,6 +54,7 @@ clone_${NAME}() { --https-url "${https_url}" \ -j ${n_jobs:-$(nproc --all --ignore=1)} \ -c checkout.defaultRemote=upstream \ + "${q[@]}" \ "${OPTS[@]}" \ -- \ "${upstream}" \ @@ -63,7 +65,7 @@ clone_${NAME}() { local upstream_branches="$(git -C "${SRC_PATH}" branch --remotes --list 'upstream/pull-request/*')"; if test -n "${upstream_branches:+x}"; then - git -C "${SRC_PATH}" branch --remotes -d ${upstream_branches}; + git -C "${SRC_PATH}" branch --remotes -d "${q[@]}" ${upstream_branches}; fi git -C "${SRC_PATH}" remote prune upstream; From c86444a81cba230db29a074fbbee53000d53a878 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 2 May 2025 07:50:31 -0700 Subject: [PATCH 14/78] print messages before doing stuff --- .../opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh | 1 + .../opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh | 1 + .../opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh | 1 + .../opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh | 1 + 4 files changed, 4 insertions(+) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh index 3185f1d88..1c1309053 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh @@ -45,6 +45,7 @@ build_${CPP_LIB}_cpp() { # Build C++ lib time ( + echo "Building lib${CPP_LIB}"; export ${CPP_ENV} PATH="$PATH"; local -a cmake_build_args="($(rapids-select-cmake-build-args ${n_jobs:+-j${n_jobs}} "${OPTS[@]}"))"; cmake \ diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh index 650976a12..7d75e7dfb 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh @@ -56,6 +56,7 @@ configure_${CPP_LIB}_cpp() { cmake_args+=(-B "${bin_dir:-${CPP_SRC}/${BIN_DIR}}"); time ( + echo "Configuring lib${CPP_LIB}"; export ${CPP_ENV} PATH="$PATH"; SCCACHE_NO_DIST_COMPILE=1 \ CUDAFLAGS="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}" \ diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh index e4dccbe3d..c0be79b7f 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh @@ -98,6 +98,7 @@ EOF trap "rm -rf '${PY_SRC}/${py_lib//"-"/"_"}.egg-info'" EXIT; time ( + echo "Building ${PY_LIB} wheel"; export ${PY_ENV} PATH="$PATH"; local cudaflags="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}"; local build_type="$(rapids-select-cmake-build-type "${cmake_args_[@]}")"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh index d1457c5bc..5c26c93f8 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh @@ -116,6 +116,7 @@ EOF trap "rm -rf '${PY_SRC}/${py_lib//"-"/"_"}.egg-info'" EXIT; time ( + echo "Installing ${PY_LIB}"; export ${PY_ENV} PATH="$PATH"; local cudaflags="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}"; local build_type="$(rapids-select-cmake-build-type "${cmake_args_[@]}")"; From 4f767f4e093a80a61eb50ad4461a7815b458d8e9 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 2 May 2025 16:51:59 -0700 Subject: [PATCH 15/78] ensure the gh cli user is refreshed if the build cluster is enabled but their token doesn't have the necessary scope --- features/src/utils/install.sh | 1 + .../opt/devcontainer/bin/github/cli/init.sh | 75 ++++++++++--------- .../devcontainer/bin/github/user/scopes.sh | 49 ++++++++++++ .../devcontainer/bin/post-attach-command.sh | 9 +-- .../opt/devcontainer/bin/sccache/dist/init.sh | 28 +++++-- .../devcontainer/bin/shell-is-interactive.sh | 2 +- 6 files changed, 115 insertions(+), 49 deletions(-) create mode 100755 features/src/utils/opt/devcontainer/bin/github/user/scopes.sh diff --git a/features/src/utils/install.sh b/features/src/utils/install.sh index 8797849a3..c7aba9239 100644 --- a/features/src/utils/install.sh +++ b/features/src/utils/install.sh @@ -102,6 +102,7 @@ declare -a commands_and_sources=( "init-ssh-deploy-keys ssh/init-deploy-keys.sh" "init-github-cli github/cli/init.sh" "clone-github-repo github/repo/clone.sh" + "github-user-scopes github/user/scopes.sh" "init-gitlab-cli gitlab/cli/init.sh" "clone-gitlab-repo gitlab/repo/clone.sh" "print-missing-gitlab-token-warning gitlab/print-missing-token-warning.sh" diff --git a/features/src/utils/opt/devcontainer/bin/github/cli/init.sh b/features/src/utils/opt/devcontainer/bin/github/cli/init.sh index 94808646f..cb2076c7b 100755 --- a/features/src/utils/opt/devcontainer/bin/github/cli/init.sh +++ b/features/src/utils/opt/devcontainer/bin/github/cli/init.sh @@ -1,5 +1,8 @@ #! /usr/bin/env bash +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/../../update-envvars.sh"; + init_github_cli() { local -; set -euo pipefail; @@ -12,7 +15,7 @@ init_github_cli() { local git_protocol="https"; local avoid_gh_cli_ssh_keygen_prompt=; - if [[ "${CODESPACES:-false}" == "true" ]]; then + if "${CODESPACES:-false}"; then git_protocol="https"; else if grep -q "You've successfully authenticated" <<< "$(ssh -T "git@${GITHUB_HOST:-github.com}" 2>&1)"; then @@ -23,67 +26,67 @@ init_github_cli() { fi fi - local -r active_scopes="$(GITHUB_TOKEN="" \ - gh api -i -X GET --silent rate_limit \ - 2>/dev/null \ - | grep -i 'x-oauth-scopes:' \ - | cut -d' ' -f1 --complement \ - | tr -d ',' \ - )"; + read -ra needed_scopes <<< "${SCCACHE_BUCKET_GH_SCOPES:-"read:org"} ${SCCACHE_DIST_GH_SCOPES:-"read:enterprise"}"; + read -ra needed_scopes <<< "${needed_scopes[*]/#/--scopes }"; - local needed_scopes="read:org read:enterprise"; + local -a wanted_scopes="($(devcontainer-utils-github-user-scopes "${needed_scopes[@]}"))"; + read -ra wanted_scopes <<< "${wanted_scopes[*]/#/--scopes }"; - needed_scopes="$( \ - comm -23 \ - <(echo -n "${needed_scopes}" | xargs -r -n1 -d' ' echo | sort -s) \ - <(echo -n "${active_scopes}" | xargs -r -n1 -d' ' echo | sort -s) \ - )"; + local -a needed_scopes="($(devcontainer-utils-github-user-scopes "${needed_scopes[@]}" --complement))"; + read -ra needed_scopes <<< "${needed_scopes[*]/#/--scopes }"; - if [ -n "${needed_scopes}" ]; then + if test "${#needed_scopes[@]}" -gt 0; then local VAR; for VAR in GH_TOKEN GITHUB_TOKEN; do - if [[ -n "$(eval "echo \${${VAR}:-}")" ]]; then - for ENVFILE in /etc/profile "$HOME/.bashrc"; do - if [[ "$(grep -q -E "^${VAR}=$" "$ENVFILE" >/dev/null 2>&1; echo $?)" != 0 ]]; then - echo "${VAR}=" | sudo tee -a "$ENVFILE" >/dev/null || true; - fi - done - unset ${VAR}; + if test -n "${!VAR:+x}"; then + local "_${VAR}=${!VAR}"; + unset_envvar "$VAR"; + unset "$VAR"; fi done fi - read -ra scopes <<< "${active_scopes} ${needed_scopes}"; - # shellcheck disable=SC2068 if ! gh auth status >/dev/null 2>&1; then echo "Logging into GitHub..." >&2; - local -r ssh_keygen="$(which ssh-keygen || echo "")"; + local -r ssh_keygen="$(which ssh-keygen 2>/dev/null || echo)"; - if [ -n "${ssh_keygen}" ] \ - && [ -n "${avoid_gh_cli_ssh_keygen_prompt}" ]; then + if test -n "${ssh_keygen:+x}" \ + && test -n "${avoid_gh_cli_ssh_keygen_prompt:+x}"; then sudo mv "${ssh_keygen}"{,.bak} || true; fi - gh auth login \ - --web --git-protocol ${git_protocol} \ + gh auth login \ + --web --git-protocol "${git_protocol}" \ --hostname "${GITHUB_HOST:-github.com}" \ - ${scopes[@]/#/--scopes } \ + "${wanted_scopes[@]}" \ || echo "Continuing without logging into GitHub"; - if [ -n "${ssh_keygen}" ] \ - && [ -n "${avoid_gh_cli_ssh_keygen_prompt}" ]; then + if test -n "${ssh_keygen:+x}" \ + && test -n "${avoid_gh_cli_ssh_keygen_prompt:+x}"; then sudo mv "${ssh_keygen}"{.bak,} || true; fi - elif [ -n "${needed_scopes}" ]; then + elif test "${#needed_scopes[@]}" -gt 0; then echo "Logging into GitHub..." >&2; - gh auth refresh \ + gh auth refresh \ --hostname "${GITHUB_HOST:-github.com}" \ - ${scopes[@]/#/--scopes } \ + "${wanted_scopes[@]}" \ || echo "Continuing without logging into GitHub"; fi + if test "${#needed_scopes[@]}" -gt 0; then + local _VAR; + for VAR in GH_TOKEN GITHUB_TOKEN; do + local _VAR="_$VAR"; + if test -n "${!_VAR:+x}"; then + export "$VAR=${!_VAR}"; + reset_envvar "$VAR"; + unset "${_VAR}" + fi + done + fi + if gh auth status >/dev/null 2>&1; then if test "$(gh config get git_protocol --host "${GITHUB_HOST:-github.com}")" != "${git_protocol}"; then gh config set git_protocol --host "${GITHUB_HOST:-github.com}" "${git_protocol}"; @@ -105,7 +108,7 @@ init_github_cli() { github_user="$(gh api user --jq '.login // ""' 2>/dev/null || echo)"; fi - export GITHUB_USER="${github_user}"; + export GITHUB_USER="${github_user:-}"; } init_github_cli "$@"; diff --git a/features/src/utils/opt/devcontainer/bin/github/user/scopes.sh b/features/src/utils/opt/devcontainer/bin/github/user/scopes.sh new file mode 100755 index 000000000..31603300d --- /dev/null +++ b/features/src/utils/opt/devcontainer/bin/github/user/scopes.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Usage: +# devcontainer-utils-github-user-scopes [OPTION]... +# +# Report the GitHub user's active and/or desired scopes. +# +# Boolean options: +# -h,--help Print this text. +# -c,--complement Only print needed scopes that are not active. +# +# Options that require values: +# -s,--scopes Additional authentication scopes the user should have. +# Can be specified multiple times. + +_github_user_scopes() { + local -; + set -euo pipefail; + + eval "$(devcontainer-utils-parse-args "$0" "$@" <&0)"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'github github-user-scopes'; + + local -a active="($(GITHUB_TOKEN="" \ + gh api -i -X GET --silent rate_limit \ + 2>/dev/null \ + | grep -i 'x-oauth-scopes:' \ + | cut -d' ' -f1 --complement \ + | tr -d ',' \ + | tr '\r' '\n' \ + | tr '\n' ' ' \ + | tr -s '[:blank:]' \ + ))" + + local -a wanted="($( \ + comm -23 \ + <(IFS=$'\n'; echo "${scopes[*]}" | sort -s) \ + <(IFS=$'\n'; echo "${active[*]}" | sort -s) \ + ))"; + + if ! test -n "${complement:+x}"; then + wanted=("${active[@]}" "${wanted[@]}"); + fi + + echo "${wanted[*]}"; +} + +_github_user_scopes "$@" <&0; diff --git a/features/src/utils/opt/devcontainer/bin/post-attach-command.sh b/features/src/utils/opt/devcontainer/bin/post-attach-command.sh index 1002a0399..568da2c62 100755 --- a/features/src/utils/opt/devcontainer/bin/post-attach-command.sh +++ b/features/src/utils/opt/devcontainer/bin/post-attach-command.sh @@ -11,19 +11,18 @@ if ! test -n "${SKIP_DEVCONTAINER_UTILS_POST_ATTACH_COMMAND:+x}"; then # Reset sccache-dist configuration SCCACHE_DIST_URL= devcontainer-utils-configure-sccache-dist; - # Initialize the GitHub CLI with the appropriate user scopes - # and load credentials for the sccache client + # Maybe load sccache client credentials # shellcheck disable=SC1091 . devcontainer-utils-creds-s3-init; # Update sccache client configuration to enable/disable sccache-dist if test -n "${DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST:+x}"; then devcontainer-utils-install-sccache --repo trxcllnt/sccache; - if ! test -n "${SCCACHE_DIST_AUTH_TOKEN:+x}" && gh auth status >/dev/null 2>&1; then + if test -n "${SCCACHE_DIST_AUTH_TOKEN:+x}"; then + devcontainer-utils-init-sccache-dist; + else # Update ~/.config/sccache/config to use gh token auth devcontainer-utils-init-sccache-dist --enable-with-github-auth; - else - devcontainer-utils-init-sccache-dist; fi fi fi diff --git a/features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh b/features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh index e5a21d36b..11df31661 100755 --- a/features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh +++ b/features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh @@ -50,15 +50,29 @@ _init_sccache_dist() { # shellcheck disable=SC1091 . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'sccache init-sccache-dist'; - if test -n "${enable_sccache_dist:+x}"; then - # Add sccache-dist configuration - devcontainer-utils-configure-sccache-dist - <<< "${OPTS[*]@Q}"; - elif test -n "${enable_with_github_auth:+x}"; then - devcontainer-utils-configure-sccache-dist - <<< "--auth-type token --auth-token '$(gh auth token)' ${OPTS[*]@Q}"; - else + while true; do + if test -n "${enable_sccache_dist:+x}"; then + # Passthrough args to configure-sccache-dist + devcontainer-utils-configure-sccache-dist - <<< "${OPTS[*]@Q}"; + break; + fi + if test -n "${enable_with_github_auth:+x}"; then + if devcontainer-utils-shell-is-interactive; then + # Initialize the GitHub CLI with the appropriate user scopes + # shellcheck disable=SC1091 + . devcontainer-utils-init-github-cli; + fi + read -ra sccache_dist_gh_scopes <<< "${SCCACHE_DIST_GH_SCOPES:-"read:enterprise"}"; + read -ra sccache_dist_gh_scopes <<< "${sccache_dist_gh_scopes[*]/#/--scopes }"; + if grep -qE "^$" <(devcontainer-utils-github-user-scopes "${sccache_dist_gh_scopes[@]}" --complement); then + devcontainer-utils-configure-sccache-dist - <<< "--auth-type token --auth-token '$(gh auth token)' ${OPTS[*]@Q}"; + break; + fi + fi # Reset sccache-dist configuration SCCACHE_DIST_URL= devcontainer-utils-configure-sccache-dist; - fi + break; + done # Restart the sccache client with the new configuration devcontainer-utils-start-sccache; diff --git a/features/src/utils/opt/devcontainer/bin/shell-is-interactive.sh b/features/src/utils/opt/devcontainer/bin/shell-is-interactive.sh index c466a2f12..3a9235661 100755 --- a/features/src/utils/opt/devcontainer/bin/shell-is-interactive.sh +++ b/features/src/utils/opt/devcontainer/bin/shell-is-interactive.sh @@ -1,6 +1,6 @@ #! /usr/bin/env bash -if tty >/dev/null 2>&1 && (exec /dev/null 2>&1 && (exec Date: Fri, 2 May 2025 16:52:04 -0700 Subject: [PATCH 16/78] cleanup --- .../opt/rapids-build-utils/bin/make-vscode-workspace.sh | 1 - .../opt/rapids-build-utils/bin/post-start-command.sh | 9 +++++---- .../bin/gitlab/print-missing-token-warning.sh | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-vscode-workspace.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-vscode-workspace.sh index 5468a1a53..e6e72924f 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-vscode-workspace.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-vscode-workspace.sh @@ -44,7 +44,6 @@ cpp_lib_dirs() { local j=0; for ((j=0; j < ${!cpp_length:-0}; j+=1)); do - # local cpp_name="${repo}_cpp_${j}_name"; local cpp_sub_dir="${repo}_cpp_${j}_sub_dir"; echo ~/"${!repo_path:-}/${!cpp_sub_dir:-}"; done diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh index c8fa92fd1..e259609f7 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh @@ -1,11 +1,12 @@ #!/usr/bin/env bash if ! test -n "${SKIP_RAPIDS_BUILD_UTILS_POST_START_COMMAND:+x}"; then - rapids-generate-scripts; - rapids-update-build-dir-links -j; - rapids-make-vscode-workspace --update; - rapids-merge-compile-commands-json > ~/compile_commands.json; + rapids-generate-scripts & + rapids-update-build-dir-links -j & + rapids-make-vscode-workspace --update & + rapids-merge-compile-commands-json > ~/compile_commands.json & if test -n "${PYTHON_PACKAGE_MANAGER:+x}"; then rapids-make-"${PYTHON_PACKAGE_MANAGER}"-env "$@" || true; fi + wait fi diff --git a/features/src/utils/opt/devcontainer/bin/gitlab/print-missing-token-warning.sh b/features/src/utils/opt/devcontainer/bin/gitlab/print-missing-token-warning.sh index 38cfc8146..738231c3a 100755 --- a/features/src/utils/opt/devcontainer/bin/gitlab/print-missing-token-warning.sh +++ b/features/src/utils/opt/devcontainer/bin/gitlab/print-missing-token-warning.sh @@ -2,7 +2,7 @@ echo "A GitLab token is required to authenticate via GitLab CLI, but a 'GITLAB_T echo "" 1>&2; echo "Generate a token with 'api' and 'write_repository' scopes at https://${GITLAB_HOST:-gitlab.com}/-/profile/personal_access_tokens" 1>&2; -if [ "${CODESPACES:-false}" = "true" ]; then +if "${CODESPACES:-false}"; then echo "To skip this prompt in the future, add the token as a GitHub codespaces secret named 'GITLAB_TOKEN' at https://github.com/settings/codespaces." 1>&2; echo " ** Be sure to allow the repo that launched this codespace access to the new 'GITLAB_TOKEN' secret **" 1>&2; fi From 9285ac533fed909fd046e42516895d914fd894dd Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 2 May 2025 17:14:55 -0700 Subject: [PATCH 17/78] log output --- .github/workflows/build-all-rapids-repos.yml | 4 ++-- .../bin/post-start-command.sh | 20 +++++++++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 7743c200e..8fcff4625 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -49,6 +49,8 @@ jobs: SCCACHE_DIST_MAX_RETRIES=inf SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false build_command: | + set -x; + # Install the latest sccache client devcontainer-utils-install-sccache --repo trxcllnt/sccache; @@ -71,8 +73,6 @@ jobs: # so that fix needs to be integrated into the kitware pip ninja builds. rapids-post-start-command --exclude <(echo ninja); - set -x; - # Configure all the C++ libs configure-all \ -j$(ulimit -Hn) \ diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh index e259609f7..8e779f0f8 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh @@ -1,10 +1,22 @@ #!/usr/bin/env bash if ! test -n "${SKIP_RAPIDS_BUILD_UTILS_POST_START_COMMAND:+x}"; then - rapids-generate-scripts & - rapids-update-build-dir-links -j & - rapids-make-vscode-workspace --update & - rapids-merge-compile-commands-json > ~/compile_commands.json & + time ( + rapids-generate-scripts; + { set +x; } 2>/dev/null; echo -n "rapids-generate-scripts time:"; + ) 2>&1 & true + time ( + rapids-update-build-dir-links -j; + { set +x; } 2>/dev/null; echo -n "rapids-update-build-dir-links time:"; + ) 2>&1 & true + time ( + rapids-make-vscode-workspace --update; + { set +x; } 2>/dev/null; echo -n "rapids-make-vscode-workspace -- time:"; + ) 2>&1 & true + time ( + rapids-merge-compile-commands-json > ~/compile_commands.json; + { set +x; } 2>/dev/null; echo -n "rapids-merge-compile-commands-json time:"; + ) 2>&1 & true if test -n "${PYTHON_PACKAGE_MANAGER:+x}"; then rapids-make-"${PYTHON_PACKAGE_MANAGER}"-env "$@" || true; fi From 471307ca08ed0c002b6e211dbafc9e4f0ba2a1d2 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 2 May 2025 17:33:59 -0700 Subject: [PATCH 18/78] be safer when constructing command args --- features/src/rapids-build-utils/install.sh | 11 +++++++---- .../rapids-build-utils/bin/generate-scripts.sh | 15 +++++++++------ features/src/utils/install.sh | 11 +++++++---- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/features/src/rapids-build-utils/install.sh b/features/src/rapids-build-utils/install.sh index d3c57bc77..7acf90f7c 100644 --- a/features/src/rapids-build-utils/install.sh +++ b/features/src/rapids-build-utils/install.sh @@ -96,10 +96,13 @@ done # Install bash_completion script if command -v devcontainer-utils-generate-bash-completion >/dev/null 2>&1; then - devcontainer-utils-generate-bash-completion \ - --out-file /etc/bash_completion.d/rapids-build-utils-completions \ - ${commands[@]/#/--command rapids-} \ - ; + read -ra commands <<< "${commands[*]/#/--command rapids-}"; + if test "${#commands[@]}" -gt 0; then + devcontainer-utils-generate-bash-completion \ + --out-file /etc/bash_completion.d/rapids-build-utils-completions \ + "${commands[@]}" \ + ; + fi fi find /opt/rapids-build-utils \ diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh index 5fe615818..cf1e8c3b4 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh @@ -17,12 +17,15 @@ generate_completions() { . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'generate-scripts'; readarray -t commands < <(find "${TMP_SCRIPT_DIR}"/ -maxdepth 1 -type f -exec basename {} \;); - - devcontainer-utils-generate-bash-completion \ - --out-file "$(realpath -m "${COMPLETION_FILE}")" \ - --template "$(realpath -m "${COMPLETION_TMPL}")" \ - ${commands[@]/#/--command } \ - ; + read -ra commands <<< "${commands[*]/#/--command }"; + + if test "${#commands[@]}" -gt 0; then + devcontainer-utils-generate-bash-completion \ + --out-file "$(realpath -m "${COMPLETION_FILE}")" \ + --template "$(realpath -m "${COMPLETION_TMPL}")" \ + "${commands[@]}" \ + ; + fi fi } diff --git a/features/src/utils/install.sh b/features/src/utils/install.sh index c7aba9239..3b9f8adfe 100644 --- a/features/src/utils/install.sh +++ b/features/src/utils/install.sh @@ -125,10 +125,13 @@ done declare -a commands="($(for pair in "${commands_and_sources[@]}"; do cut -d' ' -f1 <<< "${pair}"; done))"; # Install bash_completion script -devcontainer-utils-generate-bash-completion \ - --out-file /etc/bash_completion.d/devcontainer-utils-completions \ - ${commands[@]/#/--command devcontainer-utils-} \ -; +read -ra commands <<< "${commands[*]/#/--command devcontainer-utils-}"; +if test "${#commands[@]}" -gt 0; then + devcontainer-utils-generate-bash-completion \ + --out-file /etc/bash_completion.d/devcontainer-utils-completions \ + "${commands[@]}" \ + ; +fi find /opt/devcontainer \ \( -type d -exec chmod 0775 {} \; \ From 92d25ad696ca3be4014a64c2fcfeb3022a3c9f27 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 2 May 2025 17:34:12 -0700 Subject: [PATCH 19/78] don't background rapids-generate-scripts --- .github/workflows/build-all-rapids-repos.yml | 4 ++-- .../bin/post-start-command.sh | 20 ++++--------------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 8fcff4625..7743c200e 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -49,8 +49,6 @@ jobs: SCCACHE_DIST_MAX_RETRIES=inf SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false build_command: | - set -x; - # Install the latest sccache client devcontainer-utils-install-sccache --repo trxcllnt/sccache; @@ -73,6 +71,8 @@ jobs: # so that fix needs to be integrated into the kitware pip ninja builds. rapids-post-start-command --exclude <(echo ninja); + set -x; + # Configure all the C++ libs configure-all \ -j$(ulimit -Hn) \ diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh index 8e779f0f8..babc2713f 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh @@ -1,22 +1,10 @@ #!/usr/bin/env bash if ! test -n "${SKIP_RAPIDS_BUILD_UTILS_POST_START_COMMAND:+x}"; then - time ( - rapids-generate-scripts; - { set +x; } 2>/dev/null; echo -n "rapids-generate-scripts time:"; - ) 2>&1 & true - time ( - rapids-update-build-dir-links -j; - { set +x; } 2>/dev/null; echo -n "rapids-update-build-dir-links time:"; - ) 2>&1 & true - time ( - rapids-make-vscode-workspace --update; - { set +x; } 2>/dev/null; echo -n "rapids-make-vscode-workspace -- time:"; - ) 2>&1 & true - time ( - rapids-merge-compile-commands-json > ~/compile_commands.json; - { set +x; } 2>/dev/null; echo -n "rapids-merge-compile-commands-json time:"; - ) 2>&1 & true + rapids-generate-scripts; + rapids-update-build-dir-links -j & + rapids-make-vscode-workspace --update & + rapids-merge-compile-commands-json > ~/compile_commands.json & if test -n "${PYTHON_PACKAGE_MANAGER:+x}"; then rapids-make-"${PYTHON_PACKAGE_MANAGER}"-env "$@" || true; fi From a34964fb4329e1e7eee7fa0ab73d0f6687525e0c Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 2 May 2025 19:44:01 -0700 Subject: [PATCH 20/78] fix typo --- features/src/openmpi/devcontainer-feature.json | 2 +- features/src/openmpi/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/features/src/openmpi/devcontainer-feature.json b/features/src/openmpi/devcontainer-feature.json index 38b075b36..9f9efaf0e 100644 --- a/features/src/openmpi/devcontainer-feature.json +++ b/features/src/openmpi/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "OpenMPI", "id": "openmpi", - "version": "25.6.2", + "version": "25.6.3", "description": "A feature to install OpenMPI with optional CUDA and UCX support", "options": { "version": { diff --git a/features/src/openmpi/install.sh b/features/src/openmpi/install.sh index ba1a2526c..6333f19de 100755 --- a/features/src/openmpi/install.sh +++ b/features/src/openmpi/install.sh @@ -82,7 +82,7 @@ build_and_install_openmpi() { local -a cuda_args=(); if test "${ENABLE_CUDA:-}" = 1; then cuda_args+=(--with-cuda="${CUDA_HOME:-/usr/local/cuda}"); - cuda_args+=(--with-cuda-libdir="${CUDA_HOME:-/usr/local/cuda}/lib64/stubs}"); + cuda_args+=(--with-cuda-libdir="${CUDA_HOME:-/usr/local/cuda}/lib64/stubs"); fi IFS=" " read -r -a openmpi_dev_deps <<< "$(install_openmpi_deps)"; From bd8d712b2b38e16fc61e09aa6e466871eabef4a0 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 8 May 2025 14:48:45 -0700 Subject: [PATCH 21/78] default -j with no value to hard nofile limit in configure and build scripts --- .github/workflows/build-all-rapids-repos.yml | 6 ++---- .../opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh | 4 ++++ .../opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh | 6 +++++- .../opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh | 6 +++++- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 7743c200e..31538d37e 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -71,11 +71,9 @@ jobs: # so that fix needs to be integrated into the kitware pip ninja builds. rapids-post-start-command --exclude <(echo ninja); - set -x; - # Configure all the C++ libs configure-all \ - -j$(ulimit -Hn) \ + -j \ -Wno-dev \ -DBUILD_TESTS=ON \ -DBUILD_BENCHMARKS=ON \ @@ -85,7 +83,7 @@ jobs: -DBUILD_CUGRAPH_MG_TESTS=ON ; # Build all the libs - build-all -j$(ulimit -Hn); + build-all -j; # Print cache and dist stats sccache --show-adv-stats; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh index 1c1309053..a1264609f 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh @@ -39,6 +39,10 @@ build_${CPP_LIB}_cpp() { # shellcheck disable=SC1091 . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'build-all build-${NAME} build-${CPP_LIB}-cpp'; + if test ${#j[@]} -gt 0 && ! test -n "${j:+x}"; then + n_jobs="$(ulimit -Hn)"; + fi + # Reconfigure if not configured or previous configure failed local -r bin_dir="$(rapids-maybe-clean-build-dir -G"${G:-Ninja}" "$@" -- "${CPP_SRC}")"; test -e "${bin_dir:-${CPP_SRC}/${BIN_DIR}}" || configure-${CPP_LIB}-cpp "$@" <&0; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh index 7d75e7dfb..d9bd86778 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh @@ -31,7 +31,7 @@ configure_${CPP_LIB}_cpp() { local -a cmake_args_="(${CMAKE_ARGS:-})"; cmake_args_+=(${CPP_CMAKE_ARGS}); - eval "$(_parse_args --take '-G, -v,--verbose' "$@" "${cmake_args_[@]}" <&0)"; + eval "$(_parse_args --take '-G -j,--parallel -v,--verbose' "$@" "${cmake_args_[@]}" <&0)"; if [[ ! -d "${CPP_SRC}" ]]; then echo "configure-${CPP_LIB}-cpp: cannot access '${CPP_SRC}': No such directory" >&2; @@ -41,6 +41,10 @@ configure_${CPP_LIB}_cpp() { # shellcheck disable=SC1091 . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'configure-all configure-${NAME} configure-${CPP_LIB}-cpp'; + if test ${#j[@]} -gt 0 && ! test -n "${j:+x}"; then + n_jobs="$(ulimit -Hn)"; + fi + local -a cmake_args="( -G\"${G:-Ninja}\" ${cmake_args_[*]@Q} diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh index 5c26c93f8..e93acb295 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh @@ -34,7 +34,7 @@ install_${PY_LIB}_python() { local -a pip_args_=(${PIP_INSTALL_ARGS}); - eval "$(_parse_args --take '-G -e,--editable -v,--verbose' "$@" "${cmake_args_[@]}" "${pip_args_[@]}" <&0)"; + eval "$(_parse_args --take '-G -e,--editable -j,--parallel -v,--verbose' "$@" "${cmake_args_[@]}" "${pip_args_[@]}" <&0)"; if [[ ! -d "${PY_SRC}" ]]; then echo "install-${PY_LIB}-python: cannot access '${PY_SRC}': No such directory" >&2; @@ -44,6 +44,10 @@ install_${PY_LIB}_python() { # shellcheck disable=SC1091 . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'build-all build-${NAME} build-${PY_LIB}-python build-${PY_LIB}-python-editable install-all install-${NAME} install-${PY_LIB}-python'; + if test ${#j[@]} -gt 0 && ! test -n "${j:+x}"; then + n_jobs="$(ulimit -Hn)"; + fi + local -a cmake_args="( -G\"${G:-Ninja}\" ${cmake_args_[*]@Q} From 04001cc1a8bd5d87076e06f1c5fd9000c5973eca Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 9 May 2025 17:20:07 -0700 Subject: [PATCH 22/78] enable sccache keepalive --- .devcontainer/rapids.Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile index aa9bed6a0..d58ae3562 100644 --- a/.devcontainer/rapids.Dockerfile +++ b/.devcontainer/rapids.Dockerfile @@ -46,7 +46,10 @@ ENV PYTHONDONTWRITEBYTECODE="1" ENV SCCACHE_REGION="us-east-2" ENV SCCACHE_BUCKET="rapids-sccache-devs" ENV SCCACHE_DIST_CONNECT_TIMEOUT=30 -ENV SCCACHE_DIST_REQUEST_TIMEOUT=1800 +ENV SCCACHE_DIST_REQUEST_TIMEOUT=7200 +ENV SCCACHE_DIST_KEEPALIVE_ENABLED=true +ENV SCCACHE_DIST_KEEPALIVE_INTERVAL=20 +ENV SCCACHE_DIST_KEEPALIVE_TIMEOUT=600 ENV SCCACHE_DIST_URL="https://${TARGETARCH}.linux.sccache.gha-runners.nvidia.com" ENV SCCACHE_IDLE_TIMEOUT=1800 ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs" From 0bded8d44ae3d324c1ba29b9c2c10b88607d264f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 11 Jun 2025 13:37:34 -0700 Subject: [PATCH 23/78] switch from gha-runners.nvidia.com to rapids.nvidia.com --- .devcontainer/rapids.Dockerfile | 2 +- .github/workflows/build-and-test-feature.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile index d58ae3562..80876fb25 100644 --- a/.devcontainer/rapids.Dockerfile +++ b/.devcontainer/rapids.Dockerfile @@ -50,7 +50,7 @@ ENV SCCACHE_DIST_REQUEST_TIMEOUT=7200 ENV SCCACHE_DIST_KEEPALIVE_ENABLED=true ENV SCCACHE_DIST_KEEPALIVE_INTERVAL=20 ENV SCCACHE_DIST_KEEPALIVE_TIMEOUT=600 -ENV SCCACHE_DIST_URL="https://${TARGETARCH}.linux.sccache.gha-runners.nvidia.com" +ENV SCCACHE_DIST_URL="https://${TARGETARCH}.linux.sccache.rapids.nvidia.com" ENV SCCACHE_IDLE_TIMEOUT=1800 ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs" diff --git a/.github/workflows/build-and-test-feature.yml b/.github/workflows/build-and-test-feature.yml index a27d57d88..390637a86 100644 --- a/.github/workflows/build-and-test-feature.yml +++ b/.github/workflows/build-and-test-feature.yml @@ -40,4 +40,4 @@ jobs: aws_role_arn: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs' || '' }}" rw_sccache_bucket: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'rapids-sccache-devs' || '' }}" rw_sccache_region: "${{ vars.AWS_REGION }}" - sccache_dist_scheduler_url: "sccache.gha-runners.nvidia.com" + sccache_dist_scheduler_url: "sccache.rapids.nvidia.com" From abd6d4f1d38b253d5ab3948c65c03b16b8b82817 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 30 Jun 2025 13:32:33 -0700 Subject: [PATCH 24/78] enable and configure sccache-dist by default --- .devcontainer/rapids.Dockerfile | 34 ++++++++++++------- .github/workflows/build-all-rapids-repos.yml | 24 +++++-------- .../bin/get-num-archs-jobs-and-load.sh | 2 +- .../rapids-build-utils/bin/make-conda-env.sh | 6 +++- .../rapids-build-utils/bin/make-pip-env.sh | 8 ++++- .../devcontainer/bin/post-attach-command.sh | 19 +++++++++-- .../opt/devcontainer/bin/sccache/dist/init.sh | 26 +++++++------- features/test/utils/ubuntu18.04.sh | 25 ++++++-------- features/test/utils/ubuntu20.04.sh | 25 ++++++-------- features/test/utils/ubuntu22.04.sh | 25 ++++++-------- 10 files changed, 105 insertions(+), 89 deletions(-) diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile index 80876fb25..b56af3f9d 100644 --- a/.devcontainer/rapids.Dockerfile +++ b/.devcontainer/rapids.Dockerfile @@ -43,25 +43,35 @@ ENV PYTHONSAFEPATH="1" ENV PYTHONUNBUFFERED="1" ENV PYTHONDONTWRITEBYTECODE="1" +ENV HISTFILE="/home/coder/.cache/._bash_history" +ENV LIBCUDF_KERNEL_CACHE_PATH="/home/coder/cudf/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest/jitify_cache" + +### +# sccache configuration +### +ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs" ENV SCCACHE_REGION="us-east-2" ENV SCCACHE_BUCKET="rapids-sccache-devs" +# 2hr (1 minute longer than sccache-dist request timeout) +ENV SCCACHE_IDLE_TIMEOUT=7200 +ENV SCCACHE_S3_KEY_PREFIX=rapids-test-sccache-dist + +### +# sccache-dist configuration +### +# Enable sccache-dist by default +ENV DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 +# Compile locally if max retries exceeded +ENV SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=true +# Retry transient errors 4 times (for a total of 5 attempts) +ENV SCCACHE_DIST_MAX_RETRIES=4 ENV SCCACHE_DIST_CONNECT_TIMEOUT=30 -ENV SCCACHE_DIST_REQUEST_TIMEOUT=7200 +# 1hr 59min (to accommodate debug builds) +ENV SCCACHE_DIST_REQUEST_TIMEOUT=7140 ENV SCCACHE_DIST_KEEPALIVE_ENABLED=true ENV SCCACHE_DIST_KEEPALIVE_INTERVAL=20 ENV SCCACHE_DIST_KEEPALIVE_TIMEOUT=600 ENV SCCACHE_DIST_URL="https://${TARGETARCH}.linux.sccache.rapids.nvidia.com" -ENV SCCACHE_IDLE_TIMEOUT=1800 -ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs" - -ENV HISTFILE="/home/coder/.cache/._bash_history" - -ENV LIBCUDF_KERNEL_CACHE_PATH="/home/coder/cudf/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest/jitify_cache" - -# Prevent the sccache server from shutting down -ENV SCCACHE_IDLE_TIMEOUT=0 -ENV SCCACHE_SERVER_LOG="sccache=info" -ENV SCCACHE_S3_KEY_PREFIX=rapids-test-sccache-dist # Build as much in parallel as possible ENV INFER_NUM_DEVICE_ARCHITECTURES=1 diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 8eb290de2..665570e8e 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -27,7 +27,7 @@ jobs: if: needs.check-event.outputs.ok == 'true' needs: check-event secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.08 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@upload-sccache-client-logs permissions: actions: read packages: read @@ -41,24 +41,16 @@ jobs: extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN timeout-minutes: 720 - # 1. Prevent the sccache server from shutting down - # 2. Infinitely retry transient errors + # 1. Infinitely retry transient errors + # 2. Enable debug logging to track cache misses # 3. Never fallback to locally compiling + # 4. Use RAPIDS_AUX_SECRET_1 as the sccache-dist auth token env: | - SCCACHE_IDLE_TIMEOUT=0 SCCACHE_DIST_MAX_RETRIES=inf + SCCACHE_SERVER_LOG=sccache=debug SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false + SCCACHE_DIST_AUTH_TOKEN_VAR=RAPIDS_AUX_SECRET_1 build_command: | - # Install the latest sccache client - devcontainer-utils-install-sccache --repo trxcllnt/sccache; - - # Configure the sccache client to talk to the build cluster - devcontainer-utils-init-sccache-dist \ - --enable-sccache-dist - <<< " \ - --auth-type 'token' \ - --auth-token '$RAPIDS_AUX_SECRET_1' \ - "; - # Verify sccache cache location sccache --show-adv-stats; @@ -72,7 +64,7 @@ jobs: rapids-post-start-command --exclude <(echo ninja); # Configure all the C++ libs - configure-all \ + time configure-all \ -j \ -Wno-dev \ -DBUILD_TESTS=ON \ @@ -83,7 +75,7 @@ jobs: -DBUILD_CUGRAPH_MG_TESTS=ON ; # Build all the libs - build-all -j; + time build-all -j; # Print cache and dist stats sccache --show-adv-stats; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh index 1809640e8..94cdafb25 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh @@ -76,7 +76,7 @@ get_num_archs_jobs_and_load() { n_arch=$((n_arch > max_archs ? max_archs : n_arch)); fi - local n_load=$((parallel > n_cpus ? n_cpus : parallel)); + local n_load="$((parallel > n_cpus ? n_cpus : parallel))"; local n_jobs="$((parallel < 1 ? 1 : parallel))"; echo "declare n_arch=${n_arch}"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh index 6b6b98be4..f76cc3066 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh @@ -35,7 +35,11 @@ make_conda_env() { local -r new_env_path="$(realpath -m "/tmp/${env_file_name}")"; local -r old_env_path="$(realpath -m "${HOME}/.conda/envs/${env_file_name}")"; - rapids-make-conda-dependencies "${OPTS[@]}" > "${new_env_path}"; + # Create the python env without ninja. + # ninja -j10000000 fails with `ninja: FATAL: pipe: Too many open files`. + # This appears to have been fixed 13 years ago (https://github.com/ninja-build/ninja/issues/233), + # so that fix needs to be integrated into the kitware pip ninja builds. + rapids-make-conda-dependencies --exclude <(echo ninja) "${OPTS[@]}" > "${new_env_path}"; if test -f "${new_env_path}" && test "$(wc -l "${new_env_path}" | cut -d' ' -f1)" -gt 0; then diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh index 70e9c94ac..1c5d19f0c 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh @@ -45,7 +45,11 @@ make_pip_env() { local -r new_env_path="$(realpath -m "/tmp/${env_file_name}")"; local -r old_env_path="$(realpath -m "${HOME}/.local/share/venvs/${env_file_name}")"; - rapids-make-pip-dependencies "${OPTS[@]}" > "${new_env_path}"; + # Create the python env without ninja. + # ninja -j10000000 fails with `ninja: FATAL: pipe: Too many open files`. + # This appears to have been fixed 13 years ago (https://github.com/ninja-build/ninja/issues/233), + # so that fix needs to be integrated into the kitware pip ninja builds. + rapids-make-pip-dependencies --exclude <(echo ninja) "${OPTS[@]}" > "${new_env_path}"; if test -f "${new_env_path}"; then @@ -61,6 +65,7 @@ make_pip_env() { . "${HOME}/.local/share/venvs/${env_name}/bin/activate"; python -m pip install -U pip; python -m pip install "${pre[@]}" -U -r "${new_env_path}"; + python -m pip uninstall -y ninja >/dev/null 2>&1; # If the venv does exist but it's different from the generated one, # print the diff between the envs and update it elif ! diff -BNqw "${old_env_path}" "${new_env_path}" >/dev/null 2>&1; then @@ -79,6 +84,7 @@ make_pip_env() { . "${HOME}/.local/share/venvs/${env_name}/bin/activate"; python -m pip install -U pip; python -m pip install "${pre[@]}" -U -r "${new_env_path}"; + python -m pip uninstall -y ninja >/dev/null 2>&1; fi cp -a "${new_env_path}" "${old_env_path}"; diff --git a/features/src/utils/opt/devcontainer/bin/post-attach-command.sh b/features/src/utils/opt/devcontainer/bin/post-attach-command.sh index 568da2c62..06aa9dc2a 100755 --- a/features/src/utils/opt/devcontainer/bin/post-attach-command.sh +++ b/features/src/utils/opt/devcontainer/bin/post-attach-command.sh @@ -17,12 +17,25 @@ if ! test -n "${SKIP_DEVCONTAINER_UTILS_POST_ATTACH_COMMAND:+x}"; then # Update sccache client configuration to enable/disable sccache-dist if test -n "${DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST:+x}"; then - devcontainer-utils-install-sccache --repo trxcllnt/sccache; + devcontainer-utils-install-sccache \ + --repo "${SCCACHE_REPOSITORY:-trxcllnt/sccache}" \ + --version "${SCCACHE_VERSION:-latest}" \ + ; if test -n "${SCCACHE_DIST_AUTH_TOKEN:+x}"; then - devcontainer-utils-init-sccache-dist; + devcontainer-utils-init-sccache-dist \ + --enable-sccache-dist - <<< " \ + --auth-type 'token' \ + --auth-token '$SCCACHE_DIST_AUTH_TOKEN' \ + "; + elif test -n "${SCCACHE_DIST_AUTH_TOKEN_VAR:+x}"; then + devcontainer-utils-init-sccache-dist \ + --enable-sccache-dist - <<< " \ + --auth-type 'token' \ + --auth-token '${!SCCACHE_DIST_AUTH_TOKEN_VAR}' \ + "; else # Update ~/.config/sccache/config to use gh token auth - devcontainer-utils-init-sccache-dist --enable-with-github-auth; + devcontainer-utils-init-sccache-dist --enable-sccache-dist-with-github-auth; fi fi fi diff --git a/features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh b/features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh index 11df31661..5964c39fc 100755 --- a/features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh +++ b/features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh @@ -7,17 +7,17 @@ # then restart sccache to ensure the configuration is applied. # # Boolean options: -# -h,--help Print this text. -# -e,--enable-sccache-dist Enable sccache-dist. If omitted, disable sccache-dist. -# (default: false) -# --enable-with-github-auth Enable sccache-dist and configure it to use GitHub token auth. -# Shorthand for `-e --auth-type token --auth-token $(gh auth token)` -# (default: false) -# --no-local-compile-fallback Disable building locally after retrying transient sccache-dist errors. -# (default: false) -# --connection-pool Enable sccache client HTTP connection pool. -# (default: false) -# --keepalive Enable sccache client HTTP keepalive. +# -h,--help Print this text. +# -e,--enable-sccache-dist Enable sccache-dist. If omitted, disable sccache-dist. +# (default: false) +# --enable-sccache-dist-with-github-auth Enable sccache-dist and configure it to use GitHub token auth. +# Shorthand for `-e --auth-type token --auth-token $(gh auth token)` +# (default: false) +# --no-local-compile-fallback Disable building locally after retrying transient sccache-dist errors. +# (default: false) +# --connection-pool Enable sccache client HTTP connection pool. +# (default: false) +# --keepalive Enable sccache client HTTP keepalive. # # Options that require values: # --auth-type (token|oauth2_code_grant_pkce|oauth2_implicit) Auth type for build cluster auth. @@ -45,7 +45,7 @@ _init_sccache_dist() { local -; set -euo pipefail; - eval "$(devcontainer-utils-parse-args "$0" --take '-e,--enable-sccache-dist --enable-with-github-auth' "$@" <&0)"; + eval "$(devcontainer-utils-parse-args "$0" --take '-e,--enable-sccache-dist --enable-sccache-dist-with-github-auth' "$@" <&0)"; # shellcheck disable=SC1091 . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'sccache init-sccache-dist'; @@ -56,7 +56,7 @@ _init_sccache_dist() { devcontainer-utils-configure-sccache-dist - <<< "${OPTS[*]@Q}"; break; fi - if test -n "${enable_with_github_auth:+x}"; then + if test -n "${enable_sccache_dist_with_github_auth:+x}"; then if devcontainer-utils-shell-is-interactive; then # Initialize the GitHub CLI with the appropriate user scopes # shellcheck disable=SC1091 diff --git a/features/test/utils/ubuntu18.04.sh b/features/test/utils/ubuntu18.04.sh index 866cad28a..2cc9e943e 100644 --- a/features/test/utils/ubuntu18.04.sh +++ b/features/test/utils/ubuntu18.04.sh @@ -267,9 +267,7 @@ if test -n "${gh_token:-}" \ DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ devcontainer-utils-post-attach-command; - if expect_sccache_dist_auth_token_is_gh_token; then - return 1; - fi + !expect_sccache_dist_auth_token_is_gh_token; } check "does not configure sccache-dist auth when no GH_TOKEN or SCCACHE_DIST_AUTH_TOKEN" does_not_set_sccache_dist_auth_config; @@ -285,17 +283,16 @@ if test -n "${gh_token:-}" \ check "configures sccache-dist to use GH_TOKEN as auth token" uses_GH_TOKEN_as_sccache_dist_auth_token; - # uses_SCCACHE_DIST_AUTH_TOKEN_as_sccache_dist_auth_token() { - # reset_state; - # SCCACHE_DIST_AUTH_TYPE=token \ - # SCCACHE_DIST_AUTH_TOKEN="${gh_token}" \ - # DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ - # SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ - # devcontainer-utils-post-attach-command; - # expect_sccache_dist_auth_token_is_gh_token; - # } - - # check "configures sccache-dist to use SCCACHE_DIST_AUTH_TOKEN as auth token" uses_SCCACHE_DIST_AUTH_TOKEN_as_sccache_dist_auth_token; + uses_SCCACHE_DIST_AUTH_TOKEN_as_sccache_dist_auth_token() { + reset_state; + SCCACHE_DIST_AUTH_TOKEN="${gh_token}" \ + DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ + SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ + devcontainer-utils-post-attach-command; + expect_sccache_dist_auth_token_is_gh_token; + } + + check "configures sccache-dist to use SCCACHE_DIST_AUTH_TOKEN as auth token" uses_SCCACHE_DIST_AUTH_TOKEN_as_sccache_dist_auth_token; fi # Report result diff --git a/features/test/utils/ubuntu20.04.sh b/features/test/utils/ubuntu20.04.sh index 866cad28a..2cc9e943e 100644 --- a/features/test/utils/ubuntu20.04.sh +++ b/features/test/utils/ubuntu20.04.sh @@ -267,9 +267,7 @@ if test -n "${gh_token:-}" \ DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ devcontainer-utils-post-attach-command; - if expect_sccache_dist_auth_token_is_gh_token; then - return 1; - fi + !expect_sccache_dist_auth_token_is_gh_token; } check "does not configure sccache-dist auth when no GH_TOKEN or SCCACHE_DIST_AUTH_TOKEN" does_not_set_sccache_dist_auth_config; @@ -285,17 +283,16 @@ if test -n "${gh_token:-}" \ check "configures sccache-dist to use GH_TOKEN as auth token" uses_GH_TOKEN_as_sccache_dist_auth_token; - # uses_SCCACHE_DIST_AUTH_TOKEN_as_sccache_dist_auth_token() { - # reset_state; - # SCCACHE_DIST_AUTH_TYPE=token \ - # SCCACHE_DIST_AUTH_TOKEN="${gh_token}" \ - # DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ - # SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ - # devcontainer-utils-post-attach-command; - # expect_sccache_dist_auth_token_is_gh_token; - # } - - # check "configures sccache-dist to use SCCACHE_DIST_AUTH_TOKEN as auth token" uses_SCCACHE_DIST_AUTH_TOKEN_as_sccache_dist_auth_token; + uses_SCCACHE_DIST_AUTH_TOKEN_as_sccache_dist_auth_token() { + reset_state; + SCCACHE_DIST_AUTH_TOKEN="${gh_token}" \ + DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ + SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ + devcontainer-utils-post-attach-command; + expect_sccache_dist_auth_token_is_gh_token; + } + + check "configures sccache-dist to use SCCACHE_DIST_AUTH_TOKEN as auth token" uses_SCCACHE_DIST_AUTH_TOKEN_as_sccache_dist_auth_token; fi # Report result diff --git a/features/test/utils/ubuntu22.04.sh b/features/test/utils/ubuntu22.04.sh index 866cad28a..2cc9e943e 100644 --- a/features/test/utils/ubuntu22.04.sh +++ b/features/test/utils/ubuntu22.04.sh @@ -267,9 +267,7 @@ if test -n "${gh_token:-}" \ DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ devcontainer-utils-post-attach-command; - if expect_sccache_dist_auth_token_is_gh_token; then - return 1; - fi + !expect_sccache_dist_auth_token_is_gh_token; } check "does not configure sccache-dist auth when no GH_TOKEN or SCCACHE_DIST_AUTH_TOKEN" does_not_set_sccache_dist_auth_config; @@ -285,17 +283,16 @@ if test -n "${gh_token:-}" \ check "configures sccache-dist to use GH_TOKEN as auth token" uses_GH_TOKEN_as_sccache_dist_auth_token; - # uses_SCCACHE_DIST_AUTH_TOKEN_as_sccache_dist_auth_token() { - # reset_state; - # SCCACHE_DIST_AUTH_TYPE=token \ - # SCCACHE_DIST_AUTH_TOKEN="${gh_token}" \ - # DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ - # SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ - # devcontainer-utils-post-attach-command; - # expect_sccache_dist_auth_token_is_gh_token; - # } - - # check "configures sccache-dist to use SCCACHE_DIST_AUTH_TOKEN as auth token" uses_SCCACHE_DIST_AUTH_TOKEN_as_sccache_dist_auth_token; + uses_SCCACHE_DIST_AUTH_TOKEN_as_sccache_dist_auth_token() { + reset_state; + SCCACHE_DIST_AUTH_TOKEN="${gh_token}" \ + DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ + SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ + devcontainer-utils-post-attach-command; + expect_sccache_dist_auth_token_is_gh_token; + } + + check "configures sccache-dist to use SCCACHE_DIST_AUTH_TOKEN as auth token" uses_SCCACHE_DIST_AUTH_TOKEN_as_sccache_dist_auth_token; fi # Report result From 825192e002beb621ccf93764ca51a25f79ea10c3 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 30 Jun 2025 13:42:08 -0700 Subject: [PATCH 25/78] set RMM's default branch to main, fix devcontainer-utils tests --- .github/workflows/build-all-rapids-repos.yml | 8 +------- .../opt/rapids-build-utils/manifest.yaml | 2 +- features/test/utils/ubuntu18.04.sh | 2 +- features/test/utils/ubuntu20.04.sh | 2 +- features/test/utils/ubuntu22.04.sh | 2 +- 5 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 665570e8e..d82c7ad54 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -55,13 +55,7 @@ jobs: sccache --show-adv-stats; # Clone all the repos - clone-all -j$(nproc) -v -q --clone-upstream --depth 1 --single-branch --shallow-submodules --no-update-env; - - # Create the python env without ninja. - # ninja -j10000 fails with `ninja: FATAL: pipe: Too many open files`. - # This appears to have been fixed 13 years ago (https://github.com/ninja-build/ninja/issues/233), - # so that fix needs to be integrated into the kitware pip ninja builds. - rapids-post-start-command --exclude <(echo ninja); + clone-all -j$(nproc) -v -q --clone-upstream --depth 1 --single-branch --shallow-submodules; # Configure all the C++ libs time configure-all \ diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml index c31264110..c2f287ee0 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml @@ -12,7 +12,7 @@ repos: - name: rmm path: rmm - git: {<<: *git_defaults, repo: rmm} + git: {<<: *git_defaults, repo: rmm, tag: main} cpp: - name: rmm sub_dir: cpp diff --git a/features/test/utils/ubuntu18.04.sh b/features/test/utils/ubuntu18.04.sh index 2cc9e943e..894573128 100644 --- a/features/test/utils/ubuntu18.04.sh +++ b/features/test/utils/ubuntu18.04.sh @@ -267,7 +267,7 @@ if test -n "${gh_token:-}" \ DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ devcontainer-utils-post-attach-command; - !expect_sccache_dist_auth_token_is_gh_token; + test !expect_sccache_dist_auth_token_is_gh_token; } check "does not configure sccache-dist auth when no GH_TOKEN or SCCACHE_DIST_AUTH_TOKEN" does_not_set_sccache_dist_auth_config; diff --git a/features/test/utils/ubuntu20.04.sh b/features/test/utils/ubuntu20.04.sh index 2cc9e943e..894573128 100644 --- a/features/test/utils/ubuntu20.04.sh +++ b/features/test/utils/ubuntu20.04.sh @@ -267,7 +267,7 @@ if test -n "${gh_token:-}" \ DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ devcontainer-utils-post-attach-command; - !expect_sccache_dist_auth_token_is_gh_token; + test !expect_sccache_dist_auth_token_is_gh_token; } check "does not configure sccache-dist auth when no GH_TOKEN or SCCACHE_DIST_AUTH_TOKEN" does_not_set_sccache_dist_auth_config; diff --git a/features/test/utils/ubuntu22.04.sh b/features/test/utils/ubuntu22.04.sh index 2cc9e943e..894573128 100644 --- a/features/test/utils/ubuntu22.04.sh +++ b/features/test/utils/ubuntu22.04.sh @@ -267,7 +267,7 @@ if test -n "${gh_token:-}" \ DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 \ SCCACHE_DIST_URL="${sccache_dist_scheduler_url}" \ devcontainer-utils-post-attach-command; - !expect_sccache_dist_auth_token_is_gh_token; + test !expect_sccache_dist_auth_token_is_gh_token; } check "does not configure sccache-dist auth when no GH_TOKEN or SCCACHE_DIST_AUTH_TOKEN" does_not_set_sccache_dist_auth_config; From a7a960c814b4e0315b638827ca3e74afaa5c389e Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 30 Jun 2025 19:52:27 -0700 Subject: [PATCH 26/78] use cpu4 runners --- .github/workflows/build-all-rapids-repos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index d82c7ad54..7312f5ce3 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -37,7 +37,7 @@ jobs: with: arch: '["amd64", "arm64"]' cuda: '["12.8"]' - node_type: cpu8 + node_type: cpu4 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN timeout-minutes: 720 From 84ba6a33afd4d057c4089bdae2d04dcfd17d732c Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 1 Jul 2025 21:19:00 +0000 Subject: [PATCH 27/78] cpu4 -> cpu8 --- .github/workflows/build-all-rapids-repos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 7312f5ce3..d82c7ad54 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -37,7 +37,7 @@ jobs: with: arch: '["amd64", "arm64"]' cuda: '["12.8"]' - node_type: cpu4 + node_type: cpu8 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN timeout-minutes: 720 From 9b00a04557af2321faffc6443361f68ae4e6d044 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 1 Jul 2025 21:19:06 +0000 Subject: [PATCH 28/78] remove SCCACHE_S3_KEY_PREFIX --- .devcontainer/rapids.Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile index b56af3f9d..eedfc0b7c 100644 --- a/.devcontainer/rapids.Dockerfile +++ b/.devcontainer/rapids.Dockerfile @@ -54,7 +54,6 @@ ENV SCCACHE_REGION="us-east-2" ENV SCCACHE_BUCKET="rapids-sccache-devs" # 2hr (1 minute longer than sccache-dist request timeout) ENV SCCACHE_IDLE_TIMEOUT=7200 -ENV SCCACHE_S3_KEY_PREFIX=rapids-test-sccache-dist ### # sccache-dist configuration From e1adb331b2286417fe9a2064b54af629874856f5 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 1 Jul 2025 19:02:25 -0700 Subject: [PATCH 29/78] default to rapidsai/sccache --- features/src/utils/opt/devcontainer/bin/post-attach-command.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/src/utils/opt/devcontainer/bin/post-attach-command.sh b/features/src/utils/opt/devcontainer/bin/post-attach-command.sh index 06aa9dc2a..224eb5621 100755 --- a/features/src/utils/opt/devcontainer/bin/post-attach-command.sh +++ b/features/src/utils/opt/devcontainer/bin/post-attach-command.sh @@ -18,7 +18,7 @@ if ! test -n "${SKIP_DEVCONTAINER_UTILS_POST_ATTACH_COMMAND:+x}"; then # Update sccache client configuration to enable/disable sccache-dist if test -n "${DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST:+x}"; then devcontainer-utils-install-sccache \ - --repo "${SCCACHE_REPOSITORY:-trxcllnt/sccache}" \ + --repo "${SCCACHE_REPOSITORY:-rapidsai/sccache}" \ --version "${SCCACHE_VERSION:-latest}" \ ; if test -n "${SCCACHE_DIST_AUTH_TOKEN:+x}"; then From 2d5e5c75d3fa7f572cf6958fc41f022261b982d9 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 1 Jul 2025 19:02:44 -0700 Subject: [PATCH 30/78] switch back to rapidsai/shared-workflows@branch-25.08 --- .github/workflows/build-all-rapids-repos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index d82c7ad54..71c666d81 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -27,7 +27,7 @@ jobs: if: needs.check-event.outputs.ok == 'true' needs: check-event secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@upload-sccache-client-logs + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.08 permissions: actions: read packages: read From 2aa0507f28e7c8ffce8edb11dc349d94eff2c84a Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 2 Jul 2025 11:32:57 -0700 Subject: [PATCH 31/78] switch back to cp -n --- features/src/llvm/devcontainer-feature.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/features/src/llvm/devcontainer-feature.json b/features/src/llvm/devcontainer-feature.json index 8eb6c7ac3..80d734dda 100644 --- a/features/src/llvm/devcontainer-feature.json +++ b/features/src/llvm/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "LLVM compilers and tools", "id": "llvm", - "version": "25.8.0", + "version": "25.8.1", "description": "A feature to install LLVM compilers and tools", "options": { "version": { @@ -39,7 +39,7 @@ "updateContentCommand": [ "/bin/bash", "-c", - "mkdir -m 0755 -p ~/.config/clangd && cp -u /etc/skel/.config/clangd/config.yaml ~/.config/clangd/config.yaml" + "mkdir -m 0755 -p ~/.config/clangd && cp -n /etc/skel/.config/clangd/config.yaml ~/.config/clangd/config.yaml 2>/dev/null || true" ], "customizations": { "vscode": { From b21ad62abfc0b3bfbe2b2399ac2a30afd827ba3f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 2 Jul 2025 13:05:08 -0700 Subject: [PATCH 32/78] use ninja v1.12.1 --- image/.devcontainer/devcontainer.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/image/.devcontainer/devcontainer.json b/image/.devcontainer/devcontainer.json index aa17b0dde..0781c91cd 100644 --- a/image/.devcontainer/devcontainer.json +++ b/image/.devcontainer/devcontainer.json @@ -31,7 +31,9 @@ "ghcr.io/devcontainers/features/github-cli:1.0.12": {}, "./features/src/gitlab-cli": {}, "./features/src/cmake": {}, - "./features/src/ninja": {}, + "./features/src/ninja": { + "version": "1.12.1" + }, "./features/src/sccache": { "version": "0.7.7" }, From 04d23fb8eca48ee456fc7e5f2e3e249ff0f408f8 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 3 Jul 2025 09:15:15 -0700 Subject: [PATCH 33/78] recache --- .github/workflows/build-all-rapids-repos.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 71c666d81..20d477ce5 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -46,6 +46,7 @@ jobs: # 3. Never fallback to locally compiling # 4. Use RAPIDS_AUX_SECRET_1 as the sccache-dist auth token env: | + SCCACHE_RECACHE=1 SCCACHE_DIST_MAX_RETRIES=inf SCCACHE_SERVER_LOG=sccache=debug SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false From cf7347f8b0c85f32e9008d0decd325bd4a1da53f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 3 Jul 2025 21:12:12 +0000 Subject: [PATCH 34/78] don't recache --- .github/workflows/build-all-rapids-repos.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 20d477ce5..71c666d81 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -46,7 +46,6 @@ jobs: # 3. Never fallback to locally compiling # 4. Use RAPIDS_AUX_SECRET_1 as the sccache-dist auth token env: | - SCCACHE_RECACHE=1 SCCACHE_DIST_MAX_RETRIES=inf SCCACHE_SERVER_LOG=sccache=debug SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false From c976bb5f4ad04de19db34c4a3049bb81a3f4469f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 3 Jul 2025 21:13:18 +0000 Subject: [PATCH 35/78] rapids-get-cmake-build-dir should default to the latest build if no CMAKE_BUILD_TYPE is provided --- .../bin/get-cmake-build-dir.sh | 30 ++++++++---- .../bin/select-cmake-build-type.sh | 48 ++++++++++++++++++- .../bin/tmpl/python.build.wheel.tmpl.sh | 2 +- .../bin/tmpl/python.install.tmpl.sh | 2 +- 4 files changed, 70 insertions(+), 12 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-cmake-build-dir.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-cmake-build-dir.sh index 753af1908..456c87f8a 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-cmake-build-dir.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-cmake-build-dir.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Usage: -# rapids-get-cmake-build-dir [OPTIONS] [-DCMAKE_BUILD_TYPE=(Release|Debug)] [--] ... +# rapids-get-cmake-build-dir [OPTIONS] [-DCMAKE_BUILD_TYPE=(Debug|MinSizeRel|Release|RelWithDebInfo)] [--] ... # # Build a path to the build directory for a C++ or scikit-build-core Python library. # If the is not null and a valid directory, retarget the `build/(pip|conda)/cuda-X.Y.Z/latest` symlink @@ -19,7 +19,7 @@ # -h,--help Print this text. # --skip-links Don't update any symlinks # --skip-build-type Don't update the symlink pointing to the last component of -# the build dir path, i.e. "latest -> (debug|release)". +# the build dir path, i.e. "latest -> (debug|minsizerel|release|relwithdebinfo)". # # Positional arguments: # source_path The C++ or Python project source path @@ -43,18 +43,30 @@ get_cmake_build_dir() { if test -n "${src:+x}" && rapids-python-uses-scikit-build "${src}"; then echo "${src:+${src}/}$(python -c 'from skbuild import constants; print(constants.CMAKE_BUILD_DIR())')"; else - local -r type="$(rapids-select-cmake-build-type "${OPTS[@]}" "${REST[@]:1}" | tr '[:upper:]' '[:lower:]')"; local -r cuda="${CUDA_VERSION_MAJOR_MINOR:-}"; - local bin="build"; - bin+="${PYTHON_PACKAGE_MANAGER:+/${PYTHON_PACKAGE_MANAGER}}${cuda:+/cuda-${cuda}}"; + local bin="build${PYTHON_PACKAGE_MANAGER:+/${PYTHON_PACKAGE_MANAGER}}${cuda:+/cuda-${cuda}}"; + local build_type="$(rapids-select-cmake-build-type "${OPTS[@]}" "${REST[@]:1}" | tr '[:upper:]' '[:lower:]')"; + + if ! test -n "${build_type:+x}"; then + if test -L "${src}/${bin}/latest"; then + build_type="$(basename "$(realpath -m "${src}/${bin}/latest")")"; + fi + case "${build_type:-}" in + debug | minsizerel | release | relwithdebinfo) + ;; + *) + build_type="release"; + ;; + esac + fi if test -n "${src:+x}" && test -d "${src:-}"; then mkdir -p "${src}/${bin}"; - if ! test -n "${skip_links:+x}"; then + if ! test -n "${skip_links:+x}"; then if ! test -n "${skip_build_type:+x}" || ! test -L "${src}/${bin}/latest"; then - mkdir -p "${src}/${bin}/${type}"; + mkdir -p "${src}/${bin}/${build_type}"; cd "${src}/${bin}/" || exit 1; - ln -sfn "${type}" latest; + ln -sfn "${build_type}" latest; fi cd "${src}/build" || exit 1; local component; @@ -70,7 +82,7 @@ get_cmake_build_dir() { if test -n "${skip_build_type:+x}"; then echo "${src:+${src}/}${bin}/latest"; else - echo "${src:+${src}/}${bin}/${type}"; + echo "${src:+${src}/}${bin}/${build_type}"; fi fi } diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/select-cmake-build-type.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/select-cmake-build-type.sh index c48e5dcef..c986a1c41 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/select-cmake-build-type.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/select-cmake-build-type.sh @@ -1,3 +1,49 @@ #!/usr/bin/env bash -rapids-select-cmake-define CMAKE_BUILD_TYPE "$@" <&0 || echo "Release"; +# Usage: +# rapids-select-cmake-build-type [OPTIONS] [cmake_args]... +# +# Finds the -DCMAKE_BUILD_TYPE= definition from a list of input CMake arguments. +# If -DCMAKE_BUILD_TYPE= is found in , is printed to stdout. +# Otherwise, nothing is printed and the script exits with an error code. +# +# Boolean options: +# -h,--help Print this text. +# +# Positional arguments: +# [cmake_args]... The list of CMake arguments to search + +# shellcheck disable=SC1091 +. rapids-generate-docstring; + +select_cmake_build_type() { + local -; + set -euo pipefail; + + eval "$(_parse_args "$@" <&0)"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'select-cmake-build-type'; + + local build_type="$(rapids-select-cmake-define CMAKE_BUILD_TYPE "${OPTS[@]}" "${REST[@]}")"; + + case "${build_type,,}" in + debug) + echo "Debug" + ;; + minsizerel) + echo "MinSizeRel" + ;; + release) + echo "Release" + ;; + relwithdebinfo) + echo "RelWithDebInfo" + ;; + *) + return 1; + ;; + esac +} + +select_cmake_build_type "$@" <&0; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh index c0be79b7f..973f24bf2 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh @@ -101,7 +101,7 @@ EOF echo "Building ${PY_LIB} wheel"; export ${PY_ENV} PATH="$PATH"; local cudaflags="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}"; - local build_type="$(rapids-select-cmake-build-type "${cmake_args_[@]}")"; + local build_type="$(rapids-select-cmake-build-type "${cmake_args_[@]}" || echo "Release")"; local nvcc_append_flags="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }-t=${n_arch}"; CUDAFLAGS="${cudaflags}" \ diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh index e93acb295..6e0c13b0a 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh @@ -123,7 +123,7 @@ EOF echo "Installing ${PY_LIB}"; export ${PY_ENV} PATH="$PATH"; local cudaflags="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}"; - local build_type="$(rapids-select-cmake-build-type "${cmake_args_[@]}")"; + local build_type="$(rapids-select-cmake-build-type "${cmake_args_[@]}" || echo "Release")"; local nvcc_append_flags="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }-t=${n_arch}"; CUDAFLAGS="${cudaflags}" \ From bf9b96d07a79a040111c8e17dcf471356ec815f3 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 3 Jul 2025 21:16:14 +0000 Subject: [PATCH 36/78] build C++ and Python separately, use ninja -j0 in CI, revert to default -j behavior in other places --- .github/workflows/build-all-rapids-repos.yml | 10 +++++++--- .../bin/get-num-archs-jobs-and-load.sh | 2 +- .../opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh | 4 ---- .../rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh | 4 ---- .../rapids-build-utils/bin/tmpl/python.install.tmpl.sh | 4 ---- 5 files changed, 8 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 71c666d81..c79859755 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -59,7 +59,8 @@ jobs: # Configure all the C++ libs time configure-all \ - -j \ + -j0 \ + -GNinja \ -Wno-dev \ -DBUILD_TESTS=ON \ -DBUILD_BENCHMARKS=ON \ @@ -68,8 +69,11 @@ jobs: -DRAFT_COMPILE_LIBRARY=ON \ -DBUILD_CUGRAPH_MG_TESTS=ON ; - # Build all the libs - time build-all -j; + # Build all the C++ libs + time build-all-cpp -j0; + + # Build all the Python libs + time build-all-python -j0; # Print cache and dist stats sccache --show-adv-stats; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh index 94cdafb25..ad0bb2c4e 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh @@ -77,7 +77,7 @@ get_num_archs_jobs_and_load() { fi local n_load="$((parallel > n_cpus ? n_cpus : parallel))"; - local n_jobs="$((parallel < 1 ? 1 : parallel))"; + local n_jobs="$((parallel < 0 ? 1 : parallel))"; echo "declare n_arch=${n_arch}"; echo "declare n_jobs=${n_jobs}"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh index a1264609f..1c1309053 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh @@ -39,10 +39,6 @@ build_${CPP_LIB}_cpp() { # shellcheck disable=SC1091 . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'build-all build-${NAME} build-${CPP_LIB}-cpp'; - if test ${#j[@]} -gt 0 && ! test -n "${j:+x}"; then - n_jobs="$(ulimit -Hn)"; - fi - # Reconfigure if not configured or previous configure failed local -r bin_dir="$(rapids-maybe-clean-build-dir -G"${G:-Ninja}" "$@" -- "${CPP_SRC}")"; test -e "${bin_dir:-${CPP_SRC}/${BIN_DIR}}" || configure-${CPP_LIB}-cpp "$@" <&0; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh index d9bd86778..3850ad38b 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh @@ -41,10 +41,6 @@ configure_${CPP_LIB}_cpp() { # shellcheck disable=SC1091 . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'configure-all configure-${NAME} configure-${CPP_LIB}-cpp'; - if test ${#j[@]} -gt 0 && ! test -n "${j:+x}"; then - n_jobs="$(ulimit -Hn)"; - fi - local -a cmake_args="( -G\"${G:-Ninja}\" ${cmake_args_[*]@Q} diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh index 6e0c13b0a..167c44fdb 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh @@ -44,10 +44,6 @@ install_${PY_LIB}_python() { # shellcheck disable=SC1091 . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'build-all build-${NAME} build-${PY_LIB}-python build-${PY_LIB}-python-editable install-all install-${NAME} install-${PY_LIB}-python'; - if test ${#j[@]} -gt 0 && ! test -n "${j:+x}"; then - n_jobs="$(ulimit -Hn)"; - fi - local -a cmake_args="( -G\"${G:-Ninja}\" ${cmake_args_[*]@Q} From 3842d4e11cc9edf6d51127f177c123503d5b7bae Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 3 Jul 2025 21:51:11 +0000 Subject: [PATCH 37/78] translate -j0 to -j$(ulimit -Hn) for `cmake --build` --- .../rapids-build-utils/bin/get-num-archs-jobs-and-load.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh index ad0bb2c4e..709ff7ec1 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh @@ -76,8 +76,12 @@ get_num_archs_jobs_and_load() { n_arch=$((n_arch > max_archs ? max_archs : n_arch)); fi + if test "$parallel" -eq 0; then + parallel="$(ulimit -Hn)" + fi + local n_load="$((parallel > n_cpus ? n_cpus : parallel))"; - local n_jobs="$((parallel < 0 ? 1 : parallel))"; + local n_jobs="$((parallel < 1 ? 1 : parallel))"; echo "declare n_arch=${n_arch}"; echo "declare n_jobs=${n_jobs}"; From 63adf3a09faebd9403201f94722aace22c337377 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 4 Jul 2025 20:28:00 -0700 Subject: [PATCH 38/78] fix build-all-{cpp,python} --- .../bin/generate-scripts.sh | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh index cf1e8c3b4..0caa9a349 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh @@ -214,6 +214,8 @@ generate_scripts() { local -a cpp_dirs=(); local -a cpp_deps=(); + local -a py_names=(); + local -a cpp_names=(); local -a repo_names=(); local -a cloned_repos=(); local -a immediate_cpp_deps=(); @@ -261,8 +263,8 @@ generate_scripts() { cpp_path=~/"${!repo_path:-}${!cpp_sub_dir:+/${!cpp_sub_dir}}"; cpp_dirs+=("${cpp_path}"); - cpp_libs+=("${!cpp_name:-}"); cpp_name="${!cpp_name:-}"; + cpp_libs+=("${cpp_name}"); cpp_name_to_path["${cpp_name}"]="${cpp_path}"; @@ -303,6 +305,8 @@ generate_scripts() { fi done + cpp_names+=("${cpp_libs[@]}"); + for ((j=0; j < ${!py_length:-0}; j+=1)); do py_env="${repo}_python_${j}_env"; py_name="${repo}_python_${j}_name"; @@ -333,14 +337,15 @@ generate_scripts() { cpp_deps=(${inherited_cpp_deps[@]@Q} ${immediate_cpp_deps[@]@Q}); py_dirs+=("${py_path}"); - py_libs+=("${!py_name}"); + py_name="${!py_name:-}"; + py_libs+=("${py_name}"); if [[ -d ~/"${!repo_path:-}/.git" ]]; then NAME="${repo_name:-}" \ BIN_DIR="${bin_dir}" \ SRC_PATH=~/"${!repo_path:-}" \ PY_SRC="${py_path}" \ - PY_LIB="${!py_name}" \ + PY_LIB="${py_name}" \ PY_ENV="${!py_env:-}" \ CPP_DEPS="${cpp_deps[*]}" \ CPP_CMAKE_ARGS="${!py_cmake_args:-}" \ @@ -348,7 +353,9 @@ generate_scripts() { PIP_INSTALL_ARGS="${!pip_install_args:-}" \ generate_python_scripts ; fi - done; + done + + py_names+=("${py_libs[@]}"); if [[ -d ~/"${!repo_path:-}/.git" ]]; then NAME="${repo_name:-}" \ @@ -384,15 +391,22 @@ generate_scripts() { PREFIX="${script}" \ generate_all_script ; done - for kind in "cpp" "python"; do - # Generate a script to run a type of build for all repos - NAME="${cloned_repos[0]:-${repo_names[0]:-}}" \ - NAMES="${repo_names[*]@Q}" \ - SCRIPT="${kind}.build" \ - PREFIX="build" \ - SUFFIX="all-${kind}" \ - generate_all_script ; - done + + # Generate a script to run C++ builds for all repos + NAME="${cpp_names[0]:-${repo_names[0]:-}}" \ + NAMES="${cpp_names[*]@Q}" \ + SCRIPT="cpp.build" \ + PREFIX="build" \ + SUFFIX="all-cpp" \ + generate_all_script ; + + # Generate a script to run Python builds for all repos + NAME="${py_names[0]:-${repo_names[0]:-}}" \ + NAMES="${py_names[*]@Q}" \ + SCRIPT="python.build" \ + PREFIX="build" \ + SUFFIX="all-python" \ + generate_all_script ; fi } From 551e9471401d55d988de2488e31d2350c32062fa Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 4 Jul 2025 22:56:54 -0700 Subject: [PATCH 39/78] recache --- .github/workflows/build-all-rapids-repos.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 69eadba30..b2687a9f7 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -46,6 +46,7 @@ jobs: # 3. Never fallback to locally compiling # 4. Use RAPIDS_AUX_SECRET_1 as the sccache-dist auth token env: | + SCCACHE_RECACHE=1 SCCACHE_DIST_MAX_RETRIES=inf SCCACHE_SERVER_LOG=sccache=debug SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false From 745b5010a1512472e237e87797065ac8c3fcfc25 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sat, 5 Jul 2025 12:47:35 -0700 Subject: [PATCH 40/78] Revert "recache" This reverts commit 551e9471401d55d988de2488e31d2350c32062fa. --- .github/workflows/build-all-rapids-repos.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index b2687a9f7..69eadba30 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -46,7 +46,6 @@ jobs: # 3. Never fallback to locally compiling # 4. Use RAPIDS_AUX_SECRET_1 as the sccache-dist auth token env: | - SCCACHE_RECACHE=1 SCCACHE_DIST_MAX_RETRIES=inf SCCACHE_SERVER_LOG=sccache=debug SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false From af6851c28b55ef69ca78798b6ad3a991a51a37d7 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sun, 6 Jul 2025 12:34:32 -0700 Subject: [PATCH 41/78] remove --compress-mode and -static-global-template-stub from clangd args --- features/common/etc/skel/.config/clangd/config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/features/common/etc/skel/.config/clangd/config.yaml b/features/common/etc/skel/.config/clangd/config.yaml index 7d11148a5..52c36cb9d 100644 --- a/features/common/etc/skel/.config/clangd/config.yaml +++ b/features/common/etc/skel/.config/clangd/config.yaml @@ -59,3 +59,5 @@ CompileFlags: - "--expt-relaxed-constexpr" - "-forward-unknown-to-host-compiler" - "-Werror=cross-execution-space-call" + - "--compress-mode*" + - "-static-global-template-stub*" From 159a6c6f38a37b522aca8b22e5efa29284d6b04f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 7 Jul 2025 10:22:03 -0700 Subject: [PATCH 42/78] measure CI build times on 32-core runners with sccache disabled --- .github/workflows/build-all-rapids-repos.yml | 23 ++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 69eadba30..ff79fe3b3 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -37,7 +37,7 @@ jobs: with: arch: '["amd64", "arm64"]' cuda: '["12.9"]' - node_type: cpu8 + node_type: cpu32 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN timeout-minutes: 720 @@ -46,11 +46,26 @@ jobs: # 3. Never fallback to locally compiling # 4. Use RAPIDS_AUX_SECRET_1 as the sccache-dist auth token env: | + SCCACHE_BUCKET= + DISABLE_SCCACHE=1 + INFER_NUM_DEVICE_ARCHITECTURES= + DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST= + MAX_DEVICE_OBJ_TO_COMPILE_IN_PARALLEL=1 SCCACHE_DIST_MAX_RETRIES=inf SCCACHE_SERVER_LOG=sccache=debug SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false SCCACHE_DIST_AUTH_TOKEN_VAR=RAPIDS_AUX_SECRET_1 build_command: | + unset SCCACHE_BUCKET; + + source ~/.profile; + + echo "SCCACHE_BUCKET=${SCCACHE_BUCKET:-}"; + echo "DISABLE_SCCACHE=${DISABLE_SCCACHE:-}"; + echo "CMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER:-}"; + echo "CMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER:-}"; + echo "CMAKE_CUDA_COMPILER_LAUNCHER=${CMAKE_CUDA_COMPILER_LAUNCHER:-}"; + # Verify sccache cache location sccache --show-adv-stats; @@ -59,7 +74,7 @@ jobs: # Configure all the C++ libs time configure-all \ - -j0 \ + -j \ -GNinja \ -Wno-dev \ -DBUILD_TESTS=ON \ @@ -70,10 +85,10 @@ jobs: -DBUILD_CUGRAPH_MG_TESTS=ON ; # Build all the C++ libs - time build-all-cpp -j0; + time build-all-cpp -j; # Build all the Python libs - time build-all-python -j0; + time build-all-python -j; # Print cache and dist stats sccache --show-adv-stats; From 7b6c3edf2d1410e272621fadcf3230bf992df7b8 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 7 Jul 2025 14:13:01 -0700 Subject: [PATCH 43/78] measure CI build times on 8-core runners with sccache-dist --- .github/workflows/build-all-rapids-repos.yml | 26 +++++--------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index ff79fe3b3..8e40175e5 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -37,7 +37,7 @@ jobs: with: arch: '["amd64", "arm64"]' cuda: '["12.9"]' - node_type: cpu32 + node_type: cpu8 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN timeout-minutes: 720 @@ -46,26 +46,12 @@ jobs: # 3. Never fallback to locally compiling # 4. Use RAPIDS_AUX_SECRET_1 as the sccache-dist auth token env: | - SCCACHE_BUCKET= - DISABLE_SCCACHE=1 - INFER_NUM_DEVICE_ARCHITECTURES= - DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST= - MAX_DEVICE_OBJ_TO_COMPILE_IN_PARALLEL=1 + SCCACHE_RECACHE=1 SCCACHE_DIST_MAX_RETRIES=inf - SCCACHE_SERVER_LOG=sccache=debug + SCCACHE_SERVER_LOG=sccache=info SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false SCCACHE_DIST_AUTH_TOKEN_VAR=RAPIDS_AUX_SECRET_1 build_command: | - unset SCCACHE_BUCKET; - - source ~/.profile; - - echo "SCCACHE_BUCKET=${SCCACHE_BUCKET:-}"; - echo "DISABLE_SCCACHE=${DISABLE_SCCACHE:-}"; - echo "CMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER:-}"; - echo "CMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER:-}"; - echo "CMAKE_CUDA_COMPILER_LAUNCHER=${CMAKE_CUDA_COMPILER_LAUNCHER:-}"; - # Verify sccache cache location sccache --show-adv-stats; @@ -74,7 +60,7 @@ jobs: # Configure all the C++ libs time configure-all \ - -j \ + -j0 \ -GNinja \ -Wno-dev \ -DBUILD_TESTS=ON \ @@ -85,10 +71,10 @@ jobs: -DBUILD_CUGRAPH_MG_TESTS=ON ; # Build all the C++ libs - time build-all-cpp -j; + time build-all-cpp -j0; # Build all the Python libs - time build-all-python -j; + time build-all-python -j0; # Print cache and dist stats sccache --show-adv-stats; From 4c64bbfc12fe056ad23a4de8a98f4570f0d3abd7 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 7 Jul 2025 20:00:30 -0700 Subject: [PATCH 44/78] copy common scripts --- .../src/cccl-dev/common/etc/skel/.config/clangd/config.yaml | 2 ++ features/src/cmake/common/etc/skel/.config/clangd/config.yaml | 2 ++ features/src/cuda/common/etc/skel/.config/clangd/config.yaml | 2 ++ features/src/gcc/common/etc/skel/.config/clangd/config.yaml | 2 ++ .../src/gitlab-cli/common/etc/skel/.config/clangd/config.yaml | 2 ++ features/src/llvm/common/etc/skel/.config/clangd/config.yaml | 2 ++ .../src/mambaforge/common/etc/skel/.config/clangd/config.yaml | 2 ++ features/src/ninja/common/etc/skel/.config/clangd/config.yaml | 2 ++ features/src/nvhpc/common/etc/skel/.config/clangd/config.yaml | 2 ++ features/src/oneapi/common/etc/skel/.config/clangd/config.yaml | 2 ++ features/src/openmpi/common/etc/skel/.config/clangd/config.yaml | 2 ++ .../common/etc/skel/.config/clangd/config.yaml | 2 ++ features/src/rust/common/etc/skel/.config/clangd/config.yaml | 2 ++ features/src/sccache/common/etc/skel/.config/clangd/config.yaml | 2 ++ features/src/ucx/common/etc/skel/.config/clangd/config.yaml | 2 ++ features/src/utils/common/etc/skel/.config/clangd/config.yaml | 2 ++ 16 files changed, 32 insertions(+) diff --git a/features/src/cccl-dev/common/etc/skel/.config/clangd/config.yaml b/features/src/cccl-dev/common/etc/skel/.config/clangd/config.yaml index 7d11148a5..52c36cb9d 100644 --- a/features/src/cccl-dev/common/etc/skel/.config/clangd/config.yaml +++ b/features/src/cccl-dev/common/etc/skel/.config/clangd/config.yaml @@ -59,3 +59,5 @@ CompileFlags: - "--expt-relaxed-constexpr" - "-forward-unknown-to-host-compiler" - "-Werror=cross-execution-space-call" + - "--compress-mode*" + - "-static-global-template-stub*" diff --git a/features/src/cmake/common/etc/skel/.config/clangd/config.yaml b/features/src/cmake/common/etc/skel/.config/clangd/config.yaml index 7d11148a5..52c36cb9d 100644 --- a/features/src/cmake/common/etc/skel/.config/clangd/config.yaml +++ b/features/src/cmake/common/etc/skel/.config/clangd/config.yaml @@ -59,3 +59,5 @@ CompileFlags: - "--expt-relaxed-constexpr" - "-forward-unknown-to-host-compiler" - "-Werror=cross-execution-space-call" + - "--compress-mode*" + - "-static-global-template-stub*" diff --git a/features/src/cuda/common/etc/skel/.config/clangd/config.yaml b/features/src/cuda/common/etc/skel/.config/clangd/config.yaml index 7d11148a5..52c36cb9d 100644 --- a/features/src/cuda/common/etc/skel/.config/clangd/config.yaml +++ b/features/src/cuda/common/etc/skel/.config/clangd/config.yaml @@ -59,3 +59,5 @@ CompileFlags: - "--expt-relaxed-constexpr" - "-forward-unknown-to-host-compiler" - "-Werror=cross-execution-space-call" + - "--compress-mode*" + - "-static-global-template-stub*" diff --git a/features/src/gcc/common/etc/skel/.config/clangd/config.yaml b/features/src/gcc/common/etc/skel/.config/clangd/config.yaml index 7d11148a5..52c36cb9d 100644 --- a/features/src/gcc/common/etc/skel/.config/clangd/config.yaml +++ b/features/src/gcc/common/etc/skel/.config/clangd/config.yaml @@ -59,3 +59,5 @@ CompileFlags: - "--expt-relaxed-constexpr" - "-forward-unknown-to-host-compiler" - "-Werror=cross-execution-space-call" + - "--compress-mode*" + - "-static-global-template-stub*" diff --git a/features/src/gitlab-cli/common/etc/skel/.config/clangd/config.yaml b/features/src/gitlab-cli/common/etc/skel/.config/clangd/config.yaml index 7d11148a5..52c36cb9d 100644 --- a/features/src/gitlab-cli/common/etc/skel/.config/clangd/config.yaml +++ b/features/src/gitlab-cli/common/etc/skel/.config/clangd/config.yaml @@ -59,3 +59,5 @@ CompileFlags: - "--expt-relaxed-constexpr" - "-forward-unknown-to-host-compiler" - "-Werror=cross-execution-space-call" + - "--compress-mode*" + - "-static-global-template-stub*" diff --git a/features/src/llvm/common/etc/skel/.config/clangd/config.yaml b/features/src/llvm/common/etc/skel/.config/clangd/config.yaml index 7d11148a5..52c36cb9d 100644 --- a/features/src/llvm/common/etc/skel/.config/clangd/config.yaml +++ b/features/src/llvm/common/etc/skel/.config/clangd/config.yaml @@ -59,3 +59,5 @@ CompileFlags: - "--expt-relaxed-constexpr" - "-forward-unknown-to-host-compiler" - "-Werror=cross-execution-space-call" + - "--compress-mode*" + - "-static-global-template-stub*" diff --git a/features/src/mambaforge/common/etc/skel/.config/clangd/config.yaml b/features/src/mambaforge/common/etc/skel/.config/clangd/config.yaml index 7d11148a5..52c36cb9d 100644 --- a/features/src/mambaforge/common/etc/skel/.config/clangd/config.yaml +++ b/features/src/mambaforge/common/etc/skel/.config/clangd/config.yaml @@ -59,3 +59,5 @@ CompileFlags: - "--expt-relaxed-constexpr" - "-forward-unknown-to-host-compiler" - "-Werror=cross-execution-space-call" + - "--compress-mode*" + - "-static-global-template-stub*" diff --git a/features/src/ninja/common/etc/skel/.config/clangd/config.yaml b/features/src/ninja/common/etc/skel/.config/clangd/config.yaml index 7d11148a5..52c36cb9d 100644 --- a/features/src/ninja/common/etc/skel/.config/clangd/config.yaml +++ b/features/src/ninja/common/etc/skel/.config/clangd/config.yaml @@ -59,3 +59,5 @@ CompileFlags: - "--expt-relaxed-constexpr" - "-forward-unknown-to-host-compiler" - "-Werror=cross-execution-space-call" + - "--compress-mode*" + - "-static-global-template-stub*" diff --git a/features/src/nvhpc/common/etc/skel/.config/clangd/config.yaml b/features/src/nvhpc/common/etc/skel/.config/clangd/config.yaml index 7d11148a5..52c36cb9d 100644 --- a/features/src/nvhpc/common/etc/skel/.config/clangd/config.yaml +++ b/features/src/nvhpc/common/etc/skel/.config/clangd/config.yaml @@ -59,3 +59,5 @@ CompileFlags: - "--expt-relaxed-constexpr" - "-forward-unknown-to-host-compiler" - "-Werror=cross-execution-space-call" + - "--compress-mode*" + - "-static-global-template-stub*" diff --git a/features/src/oneapi/common/etc/skel/.config/clangd/config.yaml b/features/src/oneapi/common/etc/skel/.config/clangd/config.yaml index 7d11148a5..52c36cb9d 100644 --- a/features/src/oneapi/common/etc/skel/.config/clangd/config.yaml +++ b/features/src/oneapi/common/etc/skel/.config/clangd/config.yaml @@ -59,3 +59,5 @@ CompileFlags: - "--expt-relaxed-constexpr" - "-forward-unknown-to-host-compiler" - "-Werror=cross-execution-space-call" + - "--compress-mode*" + - "-static-global-template-stub*" diff --git a/features/src/openmpi/common/etc/skel/.config/clangd/config.yaml b/features/src/openmpi/common/etc/skel/.config/clangd/config.yaml index 7d11148a5..52c36cb9d 100644 --- a/features/src/openmpi/common/etc/skel/.config/clangd/config.yaml +++ b/features/src/openmpi/common/etc/skel/.config/clangd/config.yaml @@ -59,3 +59,5 @@ CompileFlags: - "--expt-relaxed-constexpr" - "-forward-unknown-to-host-compiler" - "-Werror=cross-execution-space-call" + - "--compress-mode*" + - "-static-global-template-stub*" diff --git a/features/src/rapids-build-utils/common/etc/skel/.config/clangd/config.yaml b/features/src/rapids-build-utils/common/etc/skel/.config/clangd/config.yaml index 7d11148a5..52c36cb9d 100644 --- a/features/src/rapids-build-utils/common/etc/skel/.config/clangd/config.yaml +++ b/features/src/rapids-build-utils/common/etc/skel/.config/clangd/config.yaml @@ -59,3 +59,5 @@ CompileFlags: - "--expt-relaxed-constexpr" - "-forward-unknown-to-host-compiler" - "-Werror=cross-execution-space-call" + - "--compress-mode*" + - "-static-global-template-stub*" diff --git a/features/src/rust/common/etc/skel/.config/clangd/config.yaml b/features/src/rust/common/etc/skel/.config/clangd/config.yaml index 7d11148a5..52c36cb9d 100644 --- a/features/src/rust/common/etc/skel/.config/clangd/config.yaml +++ b/features/src/rust/common/etc/skel/.config/clangd/config.yaml @@ -59,3 +59,5 @@ CompileFlags: - "--expt-relaxed-constexpr" - "-forward-unknown-to-host-compiler" - "-Werror=cross-execution-space-call" + - "--compress-mode*" + - "-static-global-template-stub*" diff --git a/features/src/sccache/common/etc/skel/.config/clangd/config.yaml b/features/src/sccache/common/etc/skel/.config/clangd/config.yaml index 7d11148a5..52c36cb9d 100644 --- a/features/src/sccache/common/etc/skel/.config/clangd/config.yaml +++ b/features/src/sccache/common/etc/skel/.config/clangd/config.yaml @@ -59,3 +59,5 @@ CompileFlags: - "--expt-relaxed-constexpr" - "-forward-unknown-to-host-compiler" - "-Werror=cross-execution-space-call" + - "--compress-mode*" + - "-static-global-template-stub*" diff --git a/features/src/ucx/common/etc/skel/.config/clangd/config.yaml b/features/src/ucx/common/etc/skel/.config/clangd/config.yaml index 7d11148a5..52c36cb9d 100644 --- a/features/src/ucx/common/etc/skel/.config/clangd/config.yaml +++ b/features/src/ucx/common/etc/skel/.config/clangd/config.yaml @@ -59,3 +59,5 @@ CompileFlags: - "--expt-relaxed-constexpr" - "-forward-unknown-to-host-compiler" - "-Werror=cross-execution-space-call" + - "--compress-mode*" + - "-static-global-template-stub*" diff --git a/features/src/utils/common/etc/skel/.config/clangd/config.yaml b/features/src/utils/common/etc/skel/.config/clangd/config.yaml index 7d11148a5..52c36cb9d 100644 --- a/features/src/utils/common/etc/skel/.config/clangd/config.yaml +++ b/features/src/utils/common/etc/skel/.config/clangd/config.yaml @@ -59,3 +59,5 @@ CompileFlags: - "--expt-relaxed-constexpr" - "-forward-unknown-to-host-compiler" - "-Werror=cross-execution-space-call" + - "--compress-mode*" + - "-static-global-template-stub*" From aae691bc87d68ceb2bf8297dd39440368c85f388 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 8 Jul 2025 00:30:16 -0700 Subject: [PATCH 45/78] measure CI build times on 8-core runners with sccache and sccache-dist --- .github/workflows/build-all-rapids-repos.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 8e40175e5..336cae74a 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -46,7 +46,6 @@ jobs: # 3. Never fallback to locally compiling # 4. Use RAPIDS_AUX_SECRET_1 as the sccache-dist auth token env: | - SCCACHE_RECACHE=1 SCCACHE_DIST_MAX_RETRIES=inf SCCACHE_SERVER_LOG=sccache=info SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false From 7e1d334d0e0183884fb1b70bf67242f64c70708e Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 11 Jul 2025 13:09:08 -0700 Subject: [PATCH 46/78] use rmm branch-25.08 again --- .../src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml index 5c75830fd..0477c7d7c 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml @@ -12,7 +12,7 @@ repos: - name: rmm path: rmm - git: {<<: *git_defaults, repo: rmm, tag: main} + git: {<<: *git_defaults, repo: rmm} cpp: - name: rmm sub_dir: cpp From 4dcff98429dd659cf3fd367590e5c89942782fd6 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 16 Jul 2025 08:26:05 -0700 Subject: [PATCH 47/78] remove dead code --- .../opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh index 3850ad38b..9b5f3ec97 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh @@ -31,7 +31,7 @@ configure_${CPP_LIB}_cpp() { local -a cmake_args_="(${CMAKE_ARGS:-})"; cmake_args_+=(${CPP_CMAKE_ARGS}); - eval "$(_parse_args --take '-G -j,--parallel -v,--verbose' "$@" "${cmake_args_[@]}" <&0)"; + eval "$(_parse_args --take '-G -v,--verbose' "$@" "${cmake_args_[@]}" <&0)"; if [[ ! -d "${CPP_SRC}" ]]; then echo "configure-${CPP_LIB}-cpp: cannot access '${CPP_SRC}': No such directory" >&2; From 91f03d2d0291637b90e8846d7e9b69817ac1923f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 16 Jul 2025 09:56:31 -0700 Subject: [PATCH 48/78] support repo dependency_keys in manifest.yaml --- .github/workflows/build-all-rapids-repos.yml | 1 + .../bin/make-conda-dependencies.sh | 85 ++++++++++++------- .../rapids-build-utils/bin/make-conda-env.sh | 10 ++- .../bin/make-pip-dependencies.sh | 85 ++++++++++++------- .../opt/rapids-build-utils/manifest.yaml | 6 ++ 5 files changed, 120 insertions(+), 67 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 336cae74a..e4198eb26 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -46,6 +46,7 @@ jobs: # 3. Never fallback to locally compiling # 4. Use RAPIDS_AUX_SECRET_1 as the sccache-dist auth token env: | + CONDA_ENV_CREATE_QUIET=1 SCCACHE_DIST_MAX_RETRIES=inf SCCACHE_SERVER_LOG=sccache=info SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh index a711bce3a..380a81040 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh @@ -45,7 +45,6 @@ make_conda_dependencies() { test ${#exclude[@]} -eq 0 && exclude=(); test ${#include[@]} -eq 0 && include=(); - test ${#key[@]} -eq 0 && key=(all); test ${#matrix_entry[@]} -eq 0 && matrix_entry=(); local -a _exclude=(); @@ -79,29 +78,49 @@ make_conda_dependencies() { local conda_env_yamls=(); local i; + local j; for ((i=0; i < ${repos_length:-0}; i+=1)); do local repo="repos_${i}"; local repo_name="${repo}_name"; local repo_path="${repo}_path"; + local name="${!repo_name:-}"; + local path="${!repo_path:-}"; - if [ -f ~/"${!repo_path}/dependencies.yaml" ]; then + if test -n "${name:+x}" \ + && test -n "${path:+x}" \ + && test -f ~/"${path}/dependencies.yaml"; then - echo "Generating ${!repo_name}'s repo conda env yml" 1>&2; + echo "Generating ${name}'s repo conda env yml" 1>&2; - local repo_keys=("${key[@]}"); + local dependency_keys=("${key[@]}"); + + local repo_dependency_keys_length="${repo}_dependency_keys_length"; + for ((j=0; j < ${!repo_dependency_keys_length:-0}; j+=1)); do + local dependency_key="${repo}_dependency_keys_${j}"; + dependency_key="${!dependency_key:-}"; + if test -n "${dependency_key:+x}"; then + dependency_keys+=("${dependency_key}"); + fi + done + + if test ${#dependency_keys[@]} -eq 0; then + dependency_keys=(all); + fi + + local keys=("${dependency_keys[@]}"); local keyi; - for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do - local file="/tmp/${!repo_name}.${repo_keys[$keyi]}.env.yaml"; + for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do + local file="/tmp/${name}.${keys[$keyi]}.env.yaml"; conda_env_yamls+=("${file}"); - generate_env_yaml \ - "${file}" \ - --file-key "${repo_keys[$keyi]}" \ - --output conda \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "${matrix_selectors}" \ + generate_env_yaml \ + "${file}" \ + --file-key "${keys[$keyi]}" \ + --output conda \ + --config ~/"${path}/dependencies.yaml" \ + --matrix "${matrix_selectors}" \ ; done @@ -109,21 +128,22 @@ make_conda_dependencies() { for ((j=0; j < ${!cpp_length:-0}; j+=1)); do local cpp_name="${repo}_cpp_${j}_name"; + local cpp_name="lib${!cpp_name}"; - echo "Generating lib${!cpp_name}'s conda env yml" 1>&2; + echo "Generating ${cpp_name}'s conda env yml" 1>&2; - local repo_keys=("${key[@]/%/_lib${!cpp_name//"-"/"_"}}"); + local keys=("${dependency_keys[@]/%/_${cpp_name//"-"/"_"}}"); local keyi; - for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do - local file="/tmp/${!repo_name}.lib${!cpp_name}.${repo_keys[$keyi]}.env.yaml"; + for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do + local file="/tmp/${name}.${cpp_name}.${keys[$keyi]}.env.yaml"; conda_env_yamls+=("${file}"); - generate_env_yaml \ - "${file}" \ - --file-key "${repo_keys[$keyi]}" \ - --output conda \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "${matrix_selectors}" \ + generate_env_yaml \ + "${file}" \ + --file-key "${keys[$keyi]}" \ + --output conda \ + --config ~/"${path}/dependencies.yaml" \ + --matrix "${matrix_selectors}" \ ; done done @@ -132,21 +152,22 @@ make_conda_dependencies() { for ((j=0; j < ${!py_length:-0}; j+=1)); do local py_name="${repo}_python_${j}_name"; + local py_name="${!py_name}"; - echo "Generating ${!py_name}'s conda env yml" 1>&2; + echo "Generating ${py_name}'s conda env yml" 1>&2; - local repo_keys=("${key[@]/%/_${!py_name//"-"/"_"}}"); + local keys=("${dependency_keys[@]/%/_${py_name//"-"/"_"}}"); local keyi; - for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do - local file="/tmp/${!repo_name}.${!py_name}.${repo_keys[$keyi]}.env.yaml"; + for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do + local file="/tmp/${name}.${py_name}.${keys[$keyi]}.env.yaml"; conda_env_yamls+=("${file}"); - generate_env_yaml \ - "${file}" \ - --file-key "${repo_keys[$keyi]}" \ - --output conda \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "${matrix_selectors}" \ + generate_env_yaml \ + "${file}" \ + --file-key "${keys[$keyi]}" \ + --output conda \ + --config ~/"${path}/dependencies.yaml" \ + --matrix "${matrix_selectors}" \ ; done done diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh index f76cc3066..b9e3eefc1 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh @@ -8,6 +8,8 @@ # Boolean options: # -h,--help Print this text. # -f,--force Delete the existing conda env and recreate it from scratch. +# -q,--quiet Don't show `conda env create` progress bars. +# (default: $CONDA_ENV_CREATE_QUIET) # # @_include_value_options rapids-make-conda-dependencies -h @@ -18,11 +20,13 @@ make_conda_env() { local -; set -euo pipefail; - eval "$(_parse_args --take '-f,--force' "${@:2}" <&0)"; + eval "$(_parse_args --take '-f,--force -q,--quiet' "${@:2}" <&0)"; # shellcheck disable=SC1091 . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'make-conda-env'; + test ${#quiet[@]} -eq 0 && quiet=(${CONDA_ENV_CREATE_QUIET:+"-q"}); + local env_name="${1}"; shift; local env_file_name="${env_name}.yml"; @@ -50,7 +54,7 @@ make_conda_env() { cat "${new_env_path}"; echo ""; - conda env create -q -n "${env_name}" -f "${new_env_path}" --solver=libmamba; + conda env create "${q[@]}" -n "${env_name}" -f "${new_env_path}" --solver=libmamba; # If the conda env does exist but it's different from the generated one, # print the diff between the envs and update it elif ! diff -BNqw "${old_env_path}" "${new_env_path}" >/dev/null 2>&1; then @@ -69,7 +73,7 @@ make_conda_env() { # We mount in the package cache, so this should still be fast in most cases. rm -rf "${HOME}/.conda/envs/${env_name}"; - conda env create -q -n "${env_name}" -f "${new_env_path}" --solver=libmamba; + conda env create "${q[@]}" -n "${env_name}" -f "${new_env_path}" --solver=libmamba; fi cp -a "${new_env_path}" "${old_env_path}"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index d7938e740..be3a50e5f 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -45,7 +45,6 @@ make_pip_dependencies() { test ${#exclude[@]} -eq 0 && exclude=(); test ${#include[@]} -eq 0 && include=(); - test ${#key[@]} -eq 0 && key=(py_build py_run py_test all); test ${#matrix_entry[@]} -eq 0 && matrix_entry=(); test ${#requirement[@]} -eq 0 && requirement=(); @@ -95,29 +94,49 @@ make_pip_dependencies() { eval "$(rapids-list-repos "${OPTS[@]}")"; local i; + local j; for ((i=0; i < ${repos_length:-0}; i+=1)); do local repo="repos_${i}"; local repo_name="${repo}_name"; local repo_path="${repo}_path"; + local name="${!repo_name:-}"; + local path="${!repo_path:-}"; - if [ -f ~/"${!repo_path}/dependencies.yaml" ]; then + if test -n "${name:+x}" \ + && test -n "${path:+x}" \ + && test -f ~/"${path}/dependencies.yaml"; then - echo "Generating ${!repo_name}'s repo requirements.txt" 1>&2; + echo "Generating ${name}'s repo requirements.txt" 1>&2; - local repo_keys=("${key[@]}"); + local dependency_keys=("${key[@]}"); + + local repo_dependency_keys_length="${repo}_dependency_keys_length"; + for ((j=0; j < ${!repo_dependency_keys_length:-0}; j+=1)); do + local dependency_key="${repo}_dependency_keys_${j}"; + dependency_key="${!dependency_key:-}"; + if test -n "${dependency_key:+x}"; then + dependency_keys+=("${dependency_key}"); + fi + done + + if test ${#dependency_keys[@]} -eq 0; then + dependency_keys=(py_build py_run py_test all); + fi + + local keys=("${dependency_keys[@]}"); local keyi; - for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do - local file="/tmp/${!repo_name}.${repo_keys[$keyi]}.requirements.txt"; + for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do + local file="/tmp/${name}.${keys[$keyi]}.requirements.txt"; pip_reqs_txts+=("${file}"); - generate_requirements \ - "${file}" \ - --file-key "${repo_keys[$keyi]}" \ - --output requirements \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "${matrix_selectors}" \ + generate_requirements \ + "${file}" \ + --file-key "${keys[$keyi]}" \ + --output requirements \ + --config ~/"${path}/dependencies.yaml" \ + --matrix "${matrix_selectors}" \ ; done @@ -125,21 +144,22 @@ make_pip_dependencies() { for ((j=0; j < ${!cpp_length:-0}; j+=1)); do local cpp_name="${repo}_cpp_${j}_name"; + local cpp_name="lib${!cpp_name}"; - echo "Generating lib${!cpp_name}'s requirements.txt" 1>&2; + echo "Generating ${cpp_name}'s requirements.txt" 1>&2; - local repo_keys=("${key[@]/%/_lib${!cpp_name//"-"/"_"}}"); + local keys=("${dependency_keys[@]/%/_${cpp_name//"-"/"_"}}"); local keyi; - for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do - local file="/tmp/${!repo_name}.lib${!cpp_name}.${repo_keys[$keyi]}.requirements.txt"; + for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do + local file="/tmp/${name}.${cpp_name}.${keys[$keyi]}.requirements.txt"; pip_reqs_txts+=("${file}"); - generate_requirements \ - "${file}" \ - --file-key "${repo_keys[$keyi]}" \ - --output requirements \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "${matrix_selectors}" \ + generate_requirements \ + "${file}" \ + --file-key "${keys[$keyi]}" \ + --output requirements \ + --config ~/"${path}/dependencies.yaml" \ + --matrix "${matrix_selectors}" \ ; done done @@ -148,21 +168,22 @@ make_pip_dependencies() { for ((j=0; j < ${!py_length:-0}; j+=1)); do local py_name="${repo}_python_${j}_name"; + local py_name="${!py_name}"; - echo "Generating ${!py_name}'s requirements.txt" 1>&2; + echo "Generating ${py_name}'s requirements.txt" 1>&2; - local repo_keys=("${key[@]/%/_${!py_name//"-"/"_"}}"); + local keys=("${dependency_keys[@]/%/_${py_name//"-"/"_"}}"); local keyi; - for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do - local file="/tmp/${!repo_name}.${!py_name}.${repo_keys[$keyi]}.requirements.txt"; + for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do + local file="/tmp/${name}.${py_name}.${keys[$keyi]}.requirements.txt"; pip_reqs_txts+=("${file}"); - generate_requirements \ - "${file}" \ - --file-key "${repo_keys[$keyi]}" \ - --output requirements \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "${matrix_selectors}" \ + generate_requirements \ + "${file}" \ + --file-key "${keys[$keyi]}" \ + --output requirements \ + --config ~/"${path}/dependencies.yaml" \ + --matrix "${matrix_selectors}" \ ; done done diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml index 0477c7d7c..2d36376fc 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml @@ -177,6 +177,12 @@ repos: - name: cuml path: cuml git: {<<: *git_defaults, repo: cuml} + dependency_keys: + - cpp_all + - docs + - py_build + - py_rapids_build + - py_run cpp: - name: cuml sub_dir: cpp From 863855d598c4181a8d1cc164d5a32321aef3fa7a Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 18 Jul 2025 15:28:43 -0700 Subject: [PATCH 49/78] add clean-all-{cpp,python} scripts --- .../bin/generate-scripts.sh | 32 ++++++++-------- .../bin/post-start-command.sh | 8 ++-- .../bin/tmpl/all.cpp.clean.tmpl.sh | 37 +++++++++++++++++++ .../bin/tmpl/all.python.clean.tmpl.sh | 37 +++++++++++++++++++ 4 files changed, 95 insertions(+), 19 deletions(-) create mode 100755 features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.cpp.clean.tmpl.sh create mode 100755 features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.python.clean.tmpl.sh diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh index 0caa9a349..4f28645a4 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh @@ -392,21 +392,23 @@ generate_scripts() { generate_all_script ; done - # Generate a script to run C++ builds for all repos - NAME="${cpp_names[0]:-${repo_names[0]:-}}" \ - NAMES="${cpp_names[*]@Q}" \ - SCRIPT="cpp.build" \ - PREFIX="build" \ - SUFFIX="all-cpp" \ - generate_all_script ; - - # Generate a script to run Python builds for all repos - NAME="${py_names[0]:-${repo_names[0]:-}}" \ - NAMES="${py_names[*]@Q}" \ - SCRIPT="python.build" \ - PREFIX="build" \ - SUFFIX="all-python" \ - generate_all_script ; + for script in "clean" "build"; do + # Generate a script to run C++ builds for all repos + NAME="${cpp_names[0]:-${repo_names[0]:-}}" \ + NAMES="${cpp_names[*]@Q}" \ + SCRIPT="cpp.$script" \ + PREFIX="$script" \ + SUFFIX="all-cpp" \ + generate_all_script ; + + # Generate a script to run Python builds for all repos + NAME="${py_names[0]:-${repo_names[0]:-}}" \ + NAMES="${py_names[*]@Q}" \ + SCRIPT="python.$script" \ + PREFIX="$script" \ + SUFFIX="all-python" \ + generate_all_script ; + done fi } diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh index babc2713f..f919ac1c9 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash if ! test -n "${SKIP_RAPIDS_BUILD_UTILS_POST_START_COMMAND:+x}"; then - rapids-generate-scripts; - rapids-update-build-dir-links -j & - rapids-make-vscode-workspace --update & - rapids-merge-compile-commands-json > ~/compile_commands.json & + rapids-generate-scripts; + rapids-update-build-dir-links -j & + rapids-make-vscode-workspace --update & + rapids-merge-compile-commands-json > ~/compile_commands.json & if test -n "${PYTHON_PACKAGE_MANAGER:+x}"; then rapids-make-"${PYTHON_PACKAGE_MANAGER}"-env "$@" || true; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.cpp.clean.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.cpp.clean.tmpl.sh new file mode 100755 index 000000000..4cc048c8e --- /dev/null +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.cpp.clean.tmpl.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Usage: +# clean-all-cpp [OPTION]... +# +# Runs clean- for each repo in ${NAMES}. +# +# Forwards relevant arguments to each underlying script. +# +# Boolean options: +# -h,--help Print this text. +# -v,--verbose Verbose output. +# +# Options that require values: +# -j,--parallel Clean repos in parallel + +# shellcheck disable=SC1091 +. rapids-generate-docstring; + +_clean_all_cpp() { + local -; + set -euo pipefail; + + eval "$(_parse_args --take '-h,--help' "$@" <&0)"; + + eval "$(rapids-get-num-archs-jobs-and-load --archs 0 "$@")"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'clean-all clean-all-cpp'; + + echo ${NAMES} \ + | tr '[:space:]' '\0' \ + | xargs ${v:+-t} -r -0 -P${n_jobs} -I% bash -c \ + " if command -v clean-%-cpp >/dev/null 2>&1; then if ! clean-%-cpp ${OPTS[*]@Q} ${v[*]@Q}; then exit 255; fi; fi"; +} + +_clean_all_cpp "$@" <&0; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.python.clean.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.python.clean.tmpl.sh new file mode 100755 index 000000000..c53828c3b --- /dev/null +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.python.clean.tmpl.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Usage: +# clean-all-python [OPTION]... +# +# Runs clean- for each repo in ${NAMES}. +# +# Forwards relevant arguments to each underlying script. +# +# Boolean options: +# -h,--help Print this text. +# -v,--verbose Verbose output. +# +# Options that require values: +# -j,--parallel Clean repos in parallel + +# shellcheck disable=SC1091 +. rapids-generate-docstring; + +_clean_all_python() { + local -; + set -euo pipefail; + + eval "$(_parse_args --take '-h,--help' "$@" <&0)"; + + eval "$(rapids-get-num-archs-jobs-and-load --archs 0 "$@")"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'clean-all clean-all-python'; + + echo ${NAMES} \ + | tr '[:space:]' '\0' \ + | xargs ${v:+-t} -r -0 -P${n_jobs} -I% bash -c \ + " if command -v clean-%-python >/dev/null 2>&1; then if ! clean-%-python ${OPTS[*]@Q} ${v[*]@Q}; then exit 255; fi; fi"; +} + +_clean_all_python "$@" <&0; From 28ae9aa20fce1904f86cc2cbcac6accf1368f2a6 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 18 Jul 2025 15:29:37 -0700 Subject: [PATCH 50/78] fallback to SCCACHE_VERSION and SCCACHE_REPOSITORY envvars --- .../src/utils/opt/devcontainer/bin/sccache/install.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/features/src/utils/opt/devcontainer/bin/sccache/install.sh b/features/src/utils/opt/devcontainer/bin/sccache/install.sh index 4a650b862..89ba73db1 100755 --- a/features/src/utils/opt/devcontainer/bin/sccache/install.sh +++ b/features/src/utils/opt/devcontainer/bin/sccache/install.sh @@ -10,9 +10,9 @@ # # Options that require values: # --repo The GitHub repository to use. -# (default: mozilla/sccache) +# (default: ${SCCACHE_REPOSITORY:-mozilla/sccache}) # --version The sccache version to install (e.g. `0.10.0`). -# (default: latest) +# (default: ${SCCACHE_VERSION:-latest}) # @@ -90,8 +90,8 @@ _install_sccache() { # shellcheck disable=SC1091 . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'sccache install-sccache'; - local sccache_version="${version:-"latest"}"; - local github_repo="${repo:-"mozilla/sccache"}"; + local sccache_version="${version:-"${SCCACHE_VERSION:-"latest"}"}"; + local github_repo="${repo:-"${SCCACHE_REPOSITORY:-"mozilla/sccache"}"}"; _find_version_from_git_tags sccache_version "https://github.com/$github_repo" "" "" "-.*" "true"; From 29a6b8a2018f9b60f8d992044c55196000da612d Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 18 Jul 2025 18:24:09 -0700 Subject: [PATCH 51/78] -j128 --- .github/workflows/build-all-rapids-repos.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index e4198eb26..a593d6475 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -60,7 +60,7 @@ jobs: # Configure all the C++ libs time configure-all \ - -j0 \ + -j128 \ -GNinja \ -Wno-dev \ -DBUILD_TESTS=ON \ @@ -71,10 +71,10 @@ jobs: -DBUILD_CUGRAPH_MG_TESTS=ON ; # Build all the C++ libs - time build-all-cpp -j0; + time build-all-cpp -j128; # Build all the Python libs - time build-all-python -j0; + time build-all-python -j128; # Print cache and dist stats sccache --show-adv-stats; From 4fe00e505421694027d7cf4852b25d45359abd72 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 18 Jul 2025 20:22:37 -0700 Subject: [PATCH 52/78] lower nofile ulimits --- .devcontainer/cuda12.9-conda/devcontainer.json | 2 +- .devcontainer/cuda12.9-pip/devcontainer.json | 2 +- .github/workflows/build-all-rapids-repos.yml | 6 +++--- .../rapids-build-utils/bin/get-num-archs-jobs-and-load.sh | 2 +- features/src/utils/opt/devcontainer/bin/sccache/start.sh | 4 ---- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.devcontainer/cuda12.9-conda/devcontainer.json b/.devcontainer/cuda12.9-conda/devcontainer.json index 34dad9662..25842438a 100644 --- a/.devcontainer/cuda12.9-conda/devcontainer.json +++ b/.devcontainer/cuda12.9-conda/devcontainer.json @@ -13,7 +13,7 @@ "--name", "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.08-cuda12.9-conda", "--ulimit", - "nofile=1048576:1048576" + "nofile=65536:65536" ], "hostRequirements": {"gpu": "optional"}, "features": { diff --git a/.devcontainer/cuda12.9-pip/devcontainer.json b/.devcontainer/cuda12.9-pip/devcontainer.json index f94ccaeca..c7963599e 100644 --- a/.devcontainer/cuda12.9-pip/devcontainer.json +++ b/.devcontainer/cuda12.9-pip/devcontainer.json @@ -13,7 +13,7 @@ "--name", "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.08-cuda12.9-pip", "--ulimit", - "nofile=1048576:1048576" + "nofile=65536:65536" ], "hostRequirements": {"gpu": "optional"}, "features": { diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index a593d6475..e4198eb26 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -60,7 +60,7 @@ jobs: # Configure all the C++ libs time configure-all \ - -j128 \ + -j0 \ -GNinja \ -Wno-dev \ -DBUILD_TESTS=ON \ @@ -71,10 +71,10 @@ jobs: -DBUILD_CUGRAPH_MG_TESTS=ON ; # Build all the C++ libs - time build-all-cpp -j128; + time build-all-cpp -j0; # Build all the Python libs - time build-all-python -j128; + time build-all-python -j0; # Print cache and dist stats sccache --show-adv-stats; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh index 709ff7ec1..c1035c758 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh @@ -77,7 +77,7 @@ get_num_archs_jobs_and_load() { fi if test "$parallel" -eq 0; then - parallel="$(ulimit -Hn)" + parallel="$(ulimit -n)" fi local n_load="$((parallel > n_cpus ? n_cpus : parallel))"; diff --git a/features/src/utils/opt/devcontainer/bin/sccache/start.sh b/features/src/utils/opt/devcontainer/bin/sccache/start.sh index 905a5286b..ab485e598 100755 --- a/features/src/utils/opt/devcontainer/bin/sccache/start.sh +++ b/features/src/utils/opt/devcontainer/bin/sccache/start.sh @@ -39,8 +39,6 @@ _start_sccache() { if test -n "${f:-${foreground:+x}}"; then # Unset this so sccache outputs to stderr unset SCCACHE_ERROR_LOG; - # Increase the open file limit so users can do `make -j1024` - ulimit -n "$(ulimit -Hn)"; # Start the sccache server in the foreground RUST_LOG_STYLE="always" \ SCCACHE_NO_DAEMON=1 \ @@ -63,8 +61,6 @@ _start_sccache() { | tee "$logfile"; # Write the pid to the pidfile pgrep sccache | sort -n | head -n1 | tee "${pidfile}" >/dev/null; - # Increase the open file limit so users can do `make -j1024` - prlimit --nofile=$(ulimit -Hn):$(ulimit -Hn) --pid "$(cat "${pidfile}")"; echo "=== sccache logfile: $logfile ===" >&2; echo "=== sccache pidfile: $pidfile ===" >&2; fi From d15e6e0a544ffa541253fddda2484ad83db623b5 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 21 Jul 2025 14:33:40 -0700 Subject: [PATCH 53/78] background rapids-merge-compile-commands-json call --- .../opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh index 9b5f3ec97..336812a09 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh @@ -73,7 +73,7 @@ configure_${CPP_LIB}_cpp() { ; fi - rapids-merge-compile-commands-json > ~/compile_commands.json; + rapids-merge-compile-commands-json > ~/compile_commands.json & } configure_${CPP_LIB}_cpp "$@" <&0; From 78c52549f17e5d34879aa6ef42bdb02c4cba3d44 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 21 Jul 2025 14:33:46 -0700 Subject: [PATCH 54/78] fix typo --- .../opt/rapids-build-utils/bin/make-conda-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh index b9e3eefc1..52049bf77 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh @@ -25,7 +25,7 @@ make_conda_env() { # shellcheck disable=SC1091 . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'make-conda-env'; - test ${#quiet[@]} -eq 0 && quiet=(${CONDA_ENV_CREATE_QUIET:+"-q"}); + test ${#q[@]} -eq 0 && q=(${CONDA_ENV_CREATE_QUIET:+"-q"}); local env_name="${1}"; shift; local env_file_name="${env_name}.yml"; From e94b573ee0554c2a9e2ab78bf74643dc42fb4dba Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 21 Jul 2025 14:34:00 -0700 Subject: [PATCH 55/78] set process nofile ulimit again --- features/src/utils/opt/devcontainer/bin/sccache/start.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/features/src/utils/opt/devcontainer/bin/sccache/start.sh b/features/src/utils/opt/devcontainer/bin/sccache/start.sh index ab485e598..5afc3b4cf 100755 --- a/features/src/utils/opt/devcontainer/bin/sccache/start.sh +++ b/features/src/utils/opt/devcontainer/bin/sccache/start.sh @@ -39,6 +39,8 @@ _start_sccache() { if test -n "${f:-${foreground:+x}}"; then # Unset this so sccache outputs to stderr unset SCCACHE_ERROR_LOG; + # Increase the open file limit so users can do `make -j(ulimit -n)` + ulimit -n "$(ulimit -Hn)"; # Start the sccache server in the foreground RUST_LOG_STYLE="always" \ SCCACHE_NO_DAEMON=1 \ @@ -61,6 +63,8 @@ _start_sccache() { | tee "$logfile"; # Write the pid to the pidfile pgrep sccache | sort -n | head -n1 | tee "${pidfile}" >/dev/null; + # Increase the open file limit so users can do `make -j(ulimit -n)` + prlimit --nofile=$(ulimit -Hn):$(ulimit -Hn) --pid "$(cat "${pidfile}")"; echo "=== sccache logfile: $logfile ===" >&2; echo "=== sccache pidfile: $pidfile ===" >&2; fi From b315758d387ebb927aa5eb17170e73862c98f842 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 21 Jul 2025 14:34:36 -0700 Subject: [PATCH 56/78] bump devcontainer feature version --- features/src/rapids-build-utils/devcontainer-feature.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json index 28586b606..c22ddda7c 100644 --- a/features/src/rapids-build-utils/devcontainer-feature.json +++ b/features/src/rapids-build-utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVIDIA RAPIDS devcontainer build utilities", "id": "rapids-build-utils", - "version": "25.8.9", + "version": "25.8.10", "description": "A feature to install the RAPIDS devcontainer build utilities", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" From ad092434b0081d4bf5fb90fbe78264da76a841b9 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 21 Jul 2025 14:43:00 -0700 Subject: [PATCH 57/78] test always uploading sccache client logs --- .github/workflows/build-all-rapids-repos.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index e4198eb26..7566705e1 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -27,7 +27,7 @@ jobs: if: needs.check-event.outputs.ok == 'true' needs: check-event secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.08 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fea/always-upload-sccache-logs permissions: actions: read packages: read @@ -48,7 +48,7 @@ jobs: env: | CONDA_ENV_CREATE_QUIET=1 SCCACHE_DIST_MAX_RETRIES=inf - SCCACHE_SERVER_LOG=sccache=info + SCCACHE_SERVER_LOG=sccache=debug SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false SCCACHE_DIST_AUTH_TOKEN_VAR=RAPIDS_AUX_SECRET_1 build_command: | From 9acb5dd683a856c918edd39e532d9c221b63e458 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 21 Jul 2025 16:15:25 -0700 Subject: [PATCH 58/78] print memory usage every 1s --- .github/workflows/build-all-rapids-repos.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 7566705e1..d3bce80f0 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -70,6 +70,10 @@ jobs: -DRAFT_COMPILE_LIBRARY=ON \ -DBUILD_CUGRAPH_MG_TESTS=ON ; + # Print memory usage every 1s + sh -c 'free -g -h -s 1 | grep -v Swap:' & + pid=$? + # Build all the C++ libs time build-all-cpp -j0; @@ -78,3 +82,5 @@ jobs: # Print cache and dist stats sccache --show-adv-stats; + + kill -9 $pid From 04eb45b5aed0a29e83ac628c7f4d8c2f24ededa3 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 22 Jul 2025 09:04:38 -0700 Subject: [PATCH 59/78] Fix bash race conditions in rapids-make-{conda,pip}-dependencies --- .../devcontainer-feature.json | 2 +- .../bin/make-conda-dependencies.sh | 165 +++++++++--------- .../bin/make-pip-dependencies.sh | 66 ++++--- 3 files changed, 114 insertions(+), 119 deletions(-) diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json index 28586b606..c22ddda7c 100644 --- a/features/src/rapids-build-utils/devcontainer-feature.json +++ b/features/src/rapids-build-utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVIDIA RAPIDS devcontainer build utilities", "id": "rapids-build-utils", - "version": "25.8.9", + "version": "25.8.10", "description": "A feature to install the RAPIDS devcontainer build utilities", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh index 380a81040..676a03fe9 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh @@ -24,62 +24,15 @@ # shellcheck disable=SC1091 . rapids-generate-docstring; -generate_env_yaml() { - ( - (rapids-dependency-file-generator "${@:2}" 2>/dev/null || echo "name: none") \ - | (grep -v '^#' || [ "$?" == "1" ]) \ - | tee "${1}" 1>/dev/null; - ) & true +_generate_env_yaml() { + if rapids-dependency-file-generator "$@" 2>/dev/null \ + | grep -v '^#' 2>/dev/null \ + | tee "$file" 1>/dev/null; then + echo "$file" + fi } -make_conda_dependencies() { - local -; - set -euo pipefail; - - eval "$(_parse_args --skip '-m,--manifest -o,--omit --repo' "$@" <&0)"; - - eval "$(rapids-list-repos "${OPTS[@]}")"; - - # shellcheck disable=SC1091 - . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'make-conda-env make-conda-dependencies'; - - test ${#exclude[@]} -eq 0 && exclude=(); - test ${#include[@]} -eq 0 && include=(); - test ${#matrix_entry[@]} -eq 0 && matrix_entry=(); - - local -a _exclude=(); - local exc; for exc in "${exclude[@]}"; do - # append '-f' so each file's contents will be treated as a list of patterns for 'grep' - _exclude+=(-f "${exc}"); - done - - local -a _include=(); - local inc; for inc in "${include[@]}"; do - # append '-f' so each file's contents will be treated as a list of patterns for 'grep' - _include+=(-f "${inc}"); - done - - local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}"; - cuda_version="$(grep -o '^[0-9]*.[0-9]' <<< "${cuda_version}")"; - - local python_version="${PYTHON_VERSION:-$("${ORIG_PYTHON:-python3}" --version 2>&1 | cut -d' ' -f2)}"; - python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")"; - - local -a _matrix_selectors=( - arch="$(uname -m)" - cuda="${cuda_version}" - py="${python_version}" - ); - - # add extra arguments (if there are conflicts, e.g. 'py=3.10;py=3.11', it's fine... the last one will win) - test ${#matrix_entry[@]} -gt 0 && _matrix_selectors+=("${matrix_entry[@]}"); - local -r matrix_selectors=$(IFS=";"; echo "${_matrix_selectors[*]}") - - local conda_env_yamls=(); - - local i; - local j; - +_generate_env_yamls() { for ((i=0; i < ${repos_length:-0}; i+=1)); do local repo="repos_${i}"; @@ -113,15 +66,12 @@ make_conda_dependencies() { local keyi; for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do - local file="/tmp/${name}.${keys[$keyi]}.env.yaml"; - conda_env_yamls+=("${file}"); - generate_env_yaml \ - "${file}" \ + local file="${tmpdir}/${name}.${keys[$keyi]}.env.yaml"; + _generate_env_yaml \ --file-key "${keys[$keyi]}" \ --output conda \ --config ~/"${path}/dependencies.yaml" \ - --matrix "${matrix_selectors}" \ - ; + --matrix "${matrix_selectors}" & done local cpp_length="${repo}_cpp_length"; @@ -136,15 +86,12 @@ make_conda_dependencies() { local keyi; for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do - local file="/tmp/${name}.${cpp_name}.${keys[$keyi]}.env.yaml"; - conda_env_yamls+=("${file}"); - generate_env_yaml \ - "${file}" \ + local file="${tmpdir}/${name}.${cpp_name}.${keys[$keyi]}.env.yaml"; + _generate_env_yaml \ --file-key "${keys[$keyi]}" \ --output conda \ --config ~/"${path}/dependencies.yaml" \ - --matrix "${matrix_selectors}" \ - ; + --matrix "${matrix_selectors}" & done done @@ -160,30 +107,86 @@ make_conda_dependencies() { local keyi; for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do - local file="/tmp/${name}.${py_name}.${keys[$keyi]}.env.yaml"; - conda_env_yamls+=("${file}"); - generate_env_yaml \ - "${file}" \ + local file="${tmpdir}/${name}.${py_name}.${keys[$keyi]}.env.yaml"; + _generate_env_yaml \ --file-key "${keys[$keyi]}" \ --output conda \ --config ~/"${path}/dependencies.yaml" \ - --matrix "${matrix_selectors}" \ - ; + --matrix "${matrix_selectors}" & done done fi done +} + +_make_conda_dependencies() { + local -; + set -euo pipefail; + + eval "$(_parse_args --skip '-m,--manifest -o,--omit --repo' "$@" <&0)"; + + eval "$(rapids-list-repos "${OPTS[@]}")"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'make-conda-env make-conda-dependencies'; + + test ${#exclude[@]} -eq 0 && exclude=(); + test ${#include[@]} -eq 0 && include=(); + test ${#matrix_entry[@]} -eq 0 && matrix_entry=(); + + local -a _exclude=(); + local exc; for exc in "${exclude[@]}"; do + # append '-f' so each file's contents will be treated as a list of patterns for 'grep' + _exclude+=(-f "${exc}"); + done + + local -a _include=(); + local inc; for inc in "${include[@]}"; do + # append '-f' so each file's contents will be treated as a list of patterns for 'grep' + _include+=(-f "${inc}"); + done + + local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}"; + cuda_version="$(grep -o '^[0-9]*.[0-9]' <<< "${cuda_version}")"; + + local python_version="${PYTHON_VERSION:-$("${ORIG_PYTHON:-python3}" --version 2>&1 | cut -d' ' -f2)}"; + python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")"; + + local -a _matrix_selectors=( + arch="$(uname -m)" + cuda="${cuda_version}" + py="${python_version}" + ); + + # add extra arguments (if there are conflicts, e.g. 'py=3.10;py=3.11', it's fine... the last one will win) + test ${#matrix_entry[@]} -gt 0 && _matrix_selectors+=("${matrix_entry[@]}"); + local -r matrix_selectors=$(IFS=";"; echo "${_matrix_selectors[*]}") + + local conda_env_yamls=(); + + local i; + local j; + + local -r tmpdir="$(mktemp -d)"; + # shellcheck disable=SC2064 + trap "rm -rf '${tmpdir}'" EXIT; + + readarray -t conda_env_yamls < <(_generate_env_yamls); if test ${#conda_env_yamls[@]} -gt 0; then - for ((i=0; i < ${#conda_env_yamls[@]}; i+=1)); do - while ! test -f "${conda_env_yamls[$i]}"; do - sleep 0.1; - done - done + readarray -t rapids_python_pkg_names < <(rapids-python-pkg-names); + readarray -t rapids_python_conda_pkg_names < <(rapids-python-conda-pkg-names); - # shellcheck disable=SC2207 - local conda_noinstall=($(rapids-python-pkg-names) $(rapids-python-conda-pkg-names)); + local -a conda_noinstall=(); + + conda_noinstall+=("${rapids_python_pkg_names[@]}"); + conda_noinstall+=("${rapids_python_pkg_names[@]//"-"/"_"}"); + conda_noinstall+=("${rapids_python_pkg_names[@]//"_"/"-"}"); + + conda_noinstall+=("${rapids_python_conda_pkg_names[@]}"); + conda_noinstall+=("${rapids_python_conda_pkg_names[@]//"-"/"_"}"); + conda_noinstall+=("${rapids_python_conda_pkg_names[@]//"_"/"-"}"); # Generate a combined conda env yaml file. conda-merge "${conda_env_yamls[@]}" \ @@ -193,9 +196,7 @@ make_conda_dependencies() { | ( if test ${#_exclude[@]} -gt 0; then grep -E -v "${_exclude[@]}" || [ "$?" == "1" ]; else cat -; fi ) \ | ( if test ${#_include[@]} -gt 0; then grep -E "${_include[@]}" || [ "$?" == "1" ]; else cat -; fi ) \ ; - - rm -f "${conda_env_yamls[@]}"; fi } -make_conda_dependencies "$@" <&0; +_make_conda_dependencies "$@" <&0; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index be3a50e5f..a9ea3e83e 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -26,15 +26,13 @@ # shellcheck disable=SC1091 . rapids-generate-docstring; -generate_requirements() { - ( - (rapids-dependency-file-generator "${@:2}" 2>/dev/null || echo "") \ - | (grep -v '^#' || [ "$?" == "1" ]) \ - | tee "${1}" 1>/dev/null; - ) & true +_generate_requirements() { + (rapids-dependency-file-generator "$@" 2>/dev/null || echo -n) \ + | (grep -v '^#' || [ "$?" == "1" ]) \ + | tee -a "${reqs}" 1>/dev/null; } -make_pip_dependencies() { +_make_pip_dependencies() { local -; set -euo pipefail; @@ -96,6 +94,16 @@ make_pip_dependencies() { local i; local j; + local -r tmpdir="$(mktemp -d)"; + # shellcheck disable=SC2064 + trap "rm -rf '${tmpdir}'" EXIT; + local reqs="${tmpdir}/requirements.txt"; + mkfifo "${reqs}"; + + if test ${#requirement[@]} -gt 0; then + cat "${requirement[@]}" | tee -a "$reqs"; + fi + for ((i=0; i < ${repos_length:-0}; i+=1)); do local repo="repos_${i}"; @@ -131,13 +139,12 @@ make_pip_dependencies() { for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do local file="/tmp/${name}.${keys[$keyi]}.requirements.txt"; pip_reqs_txts+=("${file}"); - generate_requirements \ - "${file}" \ + _generate_requirements \ --file-key "${keys[$keyi]}" \ --output requirements \ --config ~/"${path}/dependencies.yaml" \ --matrix "${matrix_selectors}" \ - ; + & done local cpp_length="${repo}_cpp_length"; @@ -154,13 +161,12 @@ make_pip_dependencies() { for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do local file="/tmp/${name}.${cpp_name}.${keys[$keyi]}.requirements.txt"; pip_reqs_txts+=("${file}"); - generate_requirements \ - "${file}" \ + _generate_requirements \ --file-key "${keys[$keyi]}" \ --output requirements \ --config ~/"${path}/dependencies.yaml" \ --matrix "${matrix_selectors}" \ - ; + & done done @@ -178,13 +184,12 @@ make_pip_dependencies() { for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do local file="/tmp/${name}.${py_name}.${keys[$keyi]}.requirements.txt"; pip_reqs_txts+=("${file}"); - generate_requirements \ - "${file}" \ + _generate_requirements \ --file-key "${keys[$keyi]}" \ --output requirements \ --config ~/"${path}/dependencies.yaml" \ --matrix "${matrix_selectors}" \ - ; + & done done fi @@ -192,27 +197,18 @@ make_pip_dependencies() { if test ${#requirement[@]} -gt 0 || test ${#pip_reqs_txts[@]} -gt 0; then - for ((i=0; i < ${#pip_reqs_txts[@]}; i+=1)); do - while ! test -f "${pip_reqs_txts[$i]}"; do - sleep 0.1; - done - done - - local pip_noinstall=(); + readarray -t rapids_python_pkg_names < <(rapids-python-pkg-names); # add all python packages to the noinstall list. - for pkg in $(rapids-python-pkg-names); do - pip_noinstall+=("${pkg}" "${pkg}-cu.*"); - if test -z "${pkg##*"-"*}"; then - pip_noinstall+=("${pkg//"-"/"_"}" "${pkg//"-"/"_"}-cu.*") - fi - if test -z "${pkg##*"_"*}"; then - pip_noinstall+=("${pkg//"_"/"-"}" "${pkg//"_"/"-"}-cu.*") - fi - done + local -a pip_noinstall=(); + pip_noinstall+=("${rapids_python_pkg_names[@]}"); + pip_noinstall+=("${rapids_python_pkg_names[@]//"-"/"_"}"); + pip_noinstall+=("${rapids_python_pkg_names[@]//"_"/"-"}"); + pip_noinstall+=("${pip_noinstall[@]/%/"-cu.*"}"); # Generate a combined requirements.txt file - cat "${requirement[@]}" "${pip_reqs_txts[@]}" \ + # shellcheck disable=SC2002 + cat "${reqs}" \ | (grep -v '^#' || [ "$?" == "1" ]) \ | (grep -v -E '^$' || [ "$?" == "1" ]) \ | ( if test -n "${no_dedupe:+x}"; then cat -; else tr -s "[:blank:]" | LC_ALL=C sort -u; fi ) \ @@ -223,9 +219,7 @@ make_pip_dependencies() { | sed -E "s/^cupy-cuda[0-9]+x/cupy-cuda${cuda_version_major}x/g" \ | sed -E "s/^cuda-python.*/cuda-python>=${cuda_version}.0,<$((cuda_version_major+1)).0a0/g" \ ; - - rm -f "${pip_reqs_txts[@]}"; fi } -make_pip_dependencies "$@" <&0; +_make_pip_dependencies "$@" <&0; From 723f2ee8a53b988de4ea1a1ca59471adbe67033b Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 22 Jul 2025 09:27:01 -0700 Subject: [PATCH 60/78] update devcontainer feature version --- features/src/rapids-build-utils/devcontainer-feature.json | 2 +- .../opt/rapids-build-utils/bin/make-conda-env.sh | 2 +- .../opt/rapids-build-utils/bin/make-pip-env.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json index c22ddda7c..d88b0ac36 100644 --- a/features/src/rapids-build-utils/devcontainer-feature.json +++ b/features/src/rapids-build-utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVIDIA RAPIDS devcontainer build utilities", "id": "rapids-build-utils", - "version": "25.8.10", + "version": "25.8.11", "description": "A feature to install the RAPIDS devcontainer build utilities", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh index 52049bf77..0a5b5e591 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh @@ -40,7 +40,7 @@ make_conda_env() { local -r old_env_path="$(realpath -m "${HOME}/.conda/envs/${env_file_name}")"; # Create the python env without ninja. - # ninja -j10000000 fails with `ninja: FATAL: pipe: Too many open files`. + # ninja -j$(ulimit -n) fails with `ninja: FATAL: pipe: Too many open files`. # This appears to have been fixed 13 years ago (https://github.com/ninja-build/ninja/issues/233), # so that fix needs to be integrated into the kitware pip ninja builds. rapids-make-conda-dependencies --exclude <(echo ninja) "${OPTS[@]}" > "${new_env_path}"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh index 1c5d19f0c..d25396792 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh @@ -46,7 +46,7 @@ make_pip_env() { local -r old_env_path="$(realpath -m "${HOME}/.local/share/venvs/${env_file_name}")"; # Create the python env without ninja. - # ninja -j10000000 fails with `ninja: FATAL: pipe: Too many open files`. + # ninja -$(ulimit -n) fails with `ninja: FATAL: pipe: Too many open files`. # This appears to have been fixed 13 years ago (https://github.com/ninja-build/ninja/issues/233), # so that fix needs to be integrated into the kitware pip ninja builds. rapids-make-pip-dependencies --exclude <(echo ninja) "${OPTS[@]}" > "${new_env_path}"; From de87de98dcec63bf11e5faeaeb461d26d7faab1f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 22 Jul 2025 09:27:10 -0700 Subject: [PATCH 61/78] increase nofile ulimit to 500k --- .devcontainer/cuda12.9-conda/devcontainer.json | 2 +- .devcontainer/cuda12.9-pip/devcontainer.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/cuda12.9-conda/devcontainer.json b/.devcontainer/cuda12.9-conda/devcontainer.json index 42c880ae5..abb079c27 100644 --- a/.devcontainer/cuda12.9-conda/devcontainer.json +++ b/.devcontainer/cuda12.9-conda/devcontainer.json @@ -13,7 +13,7 @@ "--name", "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.08-cuda12.9-conda", "--ulimit", - "nofile=65536:65536" + "nofile=500000" ], "hostRequirements": {"gpu": "optional"}, "features": { diff --git a/.devcontainer/cuda12.9-pip/devcontainer.json b/.devcontainer/cuda12.9-pip/devcontainer.json index 76b570eb8..145d1e33a 100644 --- a/.devcontainer/cuda12.9-pip/devcontainer.json +++ b/.devcontainer/cuda12.9-pip/devcontainer.json @@ -13,7 +13,7 @@ "--name", "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.08-cuda12.9-pip", "--ulimit", - "nofile=65536:65536" + "nofile=500000" ], "hostRequirements": {"gpu": "optional"}, "features": { From 3f1df88f0184b14875ab6dcfc55e52d49eb23b50 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 22 Jul 2025 09:54:39 -0700 Subject: [PATCH 62/78] Revert "print memory usage every 1s" This reverts commit 9acb5dd683a856c918edd39e532d9c221b63e458. --- .github/workflows/build-all-rapids-repos.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index d3bce80f0..7566705e1 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -70,10 +70,6 @@ jobs: -DRAFT_COMPILE_LIBRARY=ON \ -DBUILD_CUGRAPH_MG_TESTS=ON ; - # Print memory usage every 1s - sh -c 'free -g -h -s 1 | grep -v Swap:' & - pid=$? - # Build all the C++ libs time build-all-cpp -j0; @@ -82,5 +78,3 @@ jobs: # Print cache and dist stats sccache --show-adv-stats; - - kill -9 $pid From fc2a498e4776d23aadc8bcc1c281ad0218d8d115 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 22 Jul 2025 12:18:19 -0700 Subject: [PATCH 63/78] use the connection pool --- .devcontainer/rapids.Dockerfile | 1 + .github/workflows/build-all-rapids-repos.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile index eedfc0b7c..87a3bc4ba 100644 --- a/.devcontainer/rapids.Dockerfile +++ b/.devcontainer/rapids.Dockerfile @@ -65,6 +65,7 @@ ENV SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=true # Retry transient errors 4 times (for a total of 5 attempts) ENV SCCACHE_DIST_MAX_RETRIES=4 ENV SCCACHE_DIST_CONNECT_TIMEOUT=30 +ENV SCCACHE_DIST_CONNECTION_POOL=true # 1hr 59min (to accommodate debug builds) ENV SCCACHE_DIST_REQUEST_TIMEOUT=7140 ENV SCCACHE_DIST_KEEPALIVE_ENABLED=true diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 7566705e1..387ad96b7 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -27,7 +27,7 @@ jobs: if: needs.check-event.outputs.ok == 'true' needs: check-event secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fea/always-upload-sccache-logs + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.08 permissions: actions: read packages: read From f1bb1076126dabbf345f9f585b69b81ae93fc529 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 23 Jul 2025 13:44:05 -0700 Subject: [PATCH 64/78] fix rapids-make-pip-dependencies with -r flag --- .../opt/rapids-build-utils/bin/make-pip-dependencies.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index a9ea3e83e..97d64ce8b 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -101,7 +101,10 @@ _make_pip_dependencies() { mkfifo "${reqs}"; if test ${#requirement[@]} -gt 0; then - cat "${requirement[@]}" | tee -a "$reqs"; + cat "${requirement[@]}" 2>/dev/null \ + | (grep -v '^#' || [ "$?" == "1" ]) \ + | tee -a "${reqs}" 1>/dev/null \ + & fi for ((i=0; i < ${repos_length:-0}; i+=1)); do From 9e7512f6f8e72ab86da36a740e5c0ee804778d17 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 23 Jul 2025 16:36:26 -0700 Subject: [PATCH 65/78] use files again instead of a fifo --- .../bin/make-conda-dependencies.sh | 8 +- .../bin/make-pip-dependencies.sh | 170 +++++++++--------- 2 files changed, 84 insertions(+), 94 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh index 676a03fe9..e9b2ecb80 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh @@ -33,6 +33,9 @@ _generate_env_yaml() { } _generate_env_yamls() { + local i; + local j; + for ((i=0; i < ${repos_length:-0}; i+=1)); do local repo="repos_${i}"; @@ -162,11 +165,6 @@ _make_conda_dependencies() { test ${#matrix_entry[@]} -gt 0 && _matrix_selectors+=("${matrix_entry[@]}"); local -r matrix_selectors=$(IFS=";"; echo "${_matrix_selectors[*]}") - local conda_env_yamls=(); - - local i; - local j; - local -r tmpdir="$(mktemp -d)"; # shellcheck disable=SC2064 trap "rm -rf '${tmpdir}'" EXIT; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index 97d64ce8b..11e68d0c2 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -26,87 +26,18 @@ # shellcheck disable=SC1091 . rapids-generate-docstring; -_generate_requirements() { - (rapids-dependency-file-generator "$@" 2>/dev/null || echo -n) \ - | (grep -v '^#' || [ "$?" == "1" ]) \ - | tee -a "${reqs}" 1>/dev/null; +_generate_requirements_txt() { + if rapids-dependency-file-generator "$@" 2>/dev/null \ + | grep -v '^#' 2>/dev/null \ + | tee "$file" 1>/dev/null; then + echo "$file" + fi } -_make_pip_dependencies() { - local -; - set -euo pipefail; - - eval "$(_parse_args --skip '-m,--manifest -o,--omit --repo' "$@" <&0)"; - - # shellcheck disable=SC1091 - . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'make-pip-env make-pip-dependencies'; - - test ${#exclude[@]} -eq 0 && exclude=(); - test ${#include[@]} -eq 0 && include=(); - test ${#matrix_entry[@]} -eq 0 && matrix_entry=(); - test ${#requirement[@]} -eq 0 && requirement=(); - - local -a _exclude=(); - local exc; for exc in "${exclude[@]}"; do - # append '-f' so each file's contents will be treated as a list of patterns for 'grep' - _exclude+=(-f "${exc}"); - done - - local -a _include=(); - local inc; for inc in "${include[@]}"; do - # append '-f' so each file's contents will be treated as a list of patterns for 'grep' - _include+=(-f "${inc}"); - done - - local cuda_version="${CUDA_VERSION_MAJOR_MINOR:-}"; - local -r cuda_version_major="$(cut -d'.' -f1 <<< "${cuda_version}")"; - - local python_version="${PYTHON_VERSION:-$("${ORIG_PYTHON:-python3}" --version 2>&1 | cut -d' ' -f2)}"; - python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")"; - - # Why default to cuda_suffixed=true? - # - # Projects that depend on different pip libraries across different CUDA versions - # (e.g. 'cudf' only depending on 'pynvjitlink' from CUDA 12.0 onwards), split up their - # dependency lists with 'cuda_suffixed={true,false}'. - # - # Here we want the suffixed versions (like 'pynvjitlink-cu12'). - # - # It's ok for other RAPIDS libraries to end up in this list (like 'rmm-cu12')... in builds - # where those are also being built in the devcontainer, they'll be filtered out via - # inclusion in the 'pip_noinstall' list below. - local -a _matrix_selectors=( - arch="$(uname -m)" - cuda="${cuda_version}" - cuda_suffixed=true - py="${python_version}" - use_cuda_wheels=false - ); - - # add extra arguments (if there are conflicts, e.g. 'py=3.10;py=3.11', it's fine... the last one will win) - test ${#matrix_entry[@]} -gt 0 && _matrix_selectors+=("${matrix_entry[@]}"); - local -r matrix_selectors=$(IFS=";"; echo "${_matrix_selectors[*]}") - - local pip_reqs_txts=(); - - eval "$(rapids-list-repos "${OPTS[@]}")"; - +_generate_requirements_txts() { local i; local j; - local -r tmpdir="$(mktemp -d)"; - # shellcheck disable=SC2064 - trap "rm -rf '${tmpdir}'" EXIT; - local reqs="${tmpdir}/requirements.txt"; - mkfifo "${reqs}"; - - if test ${#requirement[@]} -gt 0; then - cat "${requirement[@]}" 2>/dev/null \ - | (grep -v '^#' || [ "$?" == "1" ]) \ - | tee -a "${reqs}" 1>/dev/null \ - & - fi - for ((i=0; i < ${repos_length:-0}; i+=1)); do local repo="repos_${i}"; @@ -140,14 +71,13 @@ _make_pip_dependencies() { local keyi; for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do - local file="/tmp/${name}.${keys[$keyi]}.requirements.txt"; + local file="${tmpdir}/${name}.${keys[$keyi]}.requirements.txt"; pip_reqs_txts+=("${file}"); - _generate_requirements \ + _generate_requirements_txt \ --file-key "${keys[$keyi]}" \ --output requirements \ --config ~/"${path}/dependencies.yaml" \ - --matrix "${matrix_selectors}" \ - & + --matrix "${matrix_selectors}" & done local cpp_length="${repo}_cpp_length"; @@ -162,14 +92,13 @@ _make_pip_dependencies() { local keyi; for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do - local file="/tmp/${name}.${cpp_name}.${keys[$keyi]}.requirements.txt"; + local file="${tmpdir}/${name}.${cpp_name}.${keys[$keyi]}.requirements.txt"; pip_reqs_txts+=("${file}"); - _generate_requirements \ + _generate_requirements_txt \ --file-key "${keys[$keyi]}" \ --output requirements \ --config ~/"${path}/dependencies.yaml" \ - --matrix "${matrix_selectors}" \ - & + --matrix "${matrix_selectors}" & done done @@ -185,18 +114,81 @@ _make_pip_dependencies() { local keyi; for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do - local file="/tmp/${name}.${py_name}.${keys[$keyi]}.requirements.txt"; + local file="${tmpdir}/${name}.${py_name}.${keys[$keyi]}.requirements.txt"; pip_reqs_txts+=("${file}"); - _generate_requirements \ + _generate_requirements_txt \ --file-key "${keys[$keyi]}" \ --output requirements \ --config ~/"${path}/dependencies.yaml" \ - --matrix "${matrix_selectors}" \ - & + --matrix "${matrix_selectors}" & done done fi done +} + +_make_pip_dependencies() { + local -; + set -euo pipefail; + + eval "$(_parse_args --skip '-m,--manifest -o,--omit --repo' "$@" <&0)"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'make-pip-env make-pip-dependencies'; + + test ${#exclude[@]} -eq 0 && exclude=(); + test ${#include[@]} -eq 0 && include=(); + test ${#matrix_entry[@]} -eq 0 && matrix_entry=(); + test ${#requirement[@]} -eq 0 && requirement=(); + + local -a _exclude=(); + local exc; for exc in "${exclude[@]}"; do + # append '-f' so each file's contents will be treated as a list of patterns for 'grep' + _exclude+=(-f "${exc}"); + done + + local -a _include=(); + local inc; for inc in "${include[@]}"; do + # append '-f' so each file's contents will be treated as a list of patterns for 'grep' + _include+=(-f "${inc}"); + done + + local cuda_version="${CUDA_VERSION_MAJOR_MINOR:-}"; + local -r cuda_version_major="$(cut -d'.' -f1 <<< "${cuda_version}")"; + + local python_version="${PYTHON_VERSION:-$("${ORIG_PYTHON:-python3}" --version 2>&1 | cut -d' ' -f2)}"; + python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")"; + + # Why default to cuda_suffixed=true? + # + # Projects that depend on different pip libraries across different CUDA versions + # (e.g. 'cudf' only depending on 'pynvjitlink' from CUDA 12.0 onwards), split up their + # dependency lists with 'cuda_suffixed={true,false}'. + # + # Here we want the suffixed versions (like 'pynvjitlink-cu12'). + # + # It's ok for other RAPIDS libraries to end up in this list (like 'rmm-cu12')... in builds + # where those are also being built in the devcontainer, they'll be filtered out via + # inclusion in the 'pip_noinstall' list below. + local -a _matrix_selectors=( + arch="$(uname -m)" + cuda="${cuda_version}" + cuda_suffixed=true + py="${python_version}" + use_cuda_wheels=false + ); + + # add extra arguments (if there are conflicts, e.g. 'py=3.10;py=3.11', it's fine... the last one will win) + test ${#matrix_entry[@]} -gt 0 && _matrix_selectors+=("${matrix_entry[@]}"); + local -r matrix_selectors=$(IFS=";"; echo "${_matrix_selectors[*]}") + + local -r tmpdir="$(mktemp -d)"; + # shellcheck disable=SC2064 + trap "rm -rf '${tmpdir}'" EXIT; + + eval "$(rapids-list-repos "${OPTS[@]}")"; + + readarray -t pip_reqs_txts < <(_generate_requirements_txts); if test ${#requirement[@]} -gt 0 || test ${#pip_reqs_txts[@]} -gt 0; then @@ -211,7 +203,7 @@ _make_pip_dependencies() { # Generate a combined requirements.txt file # shellcheck disable=SC2002 - cat "${reqs}" \ + cat "${requirement[@]}" "${pip_reqs_txts[@]}" \ | (grep -v '^#' || [ "$?" == "1" ]) \ | (grep -v -E '^$' || [ "$?" == "1" ]) \ | ( if test -n "${no_dedupe:+x}"; then cat -; else tr -s "[:blank:]" | LC_ALL=C sort -u; fi ) \ From d4433669dde5db8d6304dd638247344351942032 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 29 Jul 2025 20:14:46 -0700 Subject: [PATCH 66/78] don't use connection pooling --- .devcontainer/rapids.Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile index 87a3bc4ba..545c8c1b5 100644 --- a/.devcontainer/rapids.Dockerfile +++ b/.devcontainer/rapids.Dockerfile @@ -54,6 +54,7 @@ ENV SCCACHE_REGION="us-east-2" ENV SCCACHE_BUCKET="rapids-sccache-devs" # 2hr (1 minute longer than sccache-dist request timeout) ENV SCCACHE_IDLE_TIMEOUT=7200 +ENV SCCACHE_S3_KEY_PREFIX=rapids-test-sccache-dist ### # sccache-dist configuration @@ -65,7 +66,7 @@ ENV SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=true # Retry transient errors 4 times (for a total of 5 attempts) ENV SCCACHE_DIST_MAX_RETRIES=4 ENV SCCACHE_DIST_CONNECT_TIMEOUT=30 -ENV SCCACHE_DIST_CONNECTION_POOL=true +ENV SCCACHE_DIST_CONNECTION_POOL=false # 1hr 59min (to accommodate debug builds) ENV SCCACHE_DIST_REQUEST_TIMEOUT=7140 ENV SCCACHE_DIST_KEEPALIVE_ENABLED=true From b4ae283bf9b8235d34ba13e6627cbc948506d1af Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 29 Jul 2025 20:15:10 -0700 Subject: [PATCH 67/78] persist the preprocessor cache across runs --- .github/workflows/build-all-rapids-repos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 387ad96b7..ae0a5c075 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -27,7 +27,7 @@ jobs: if: needs.check-event.outputs.ok == 'true' needs: check-event secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.08 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fea/preprocessor-caching permissions: actions: read packages: read From b48f5846a05951b4222dc3ac15d4fedf7af169cb Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 31 Jul 2025 16:10:24 -0700 Subject: [PATCH 68/78] recache --- .github/workflows/build-all-rapids-repos.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index ae0a5c075..2a4dc0057 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -46,6 +46,7 @@ jobs: # 3. Never fallback to locally compiling # 4. Use RAPIDS_AUX_SECRET_1 as the sccache-dist auth token env: | + SCCACHE_RECACHE=1 CONDA_ENV_CREATE_QUIET=1 SCCACHE_DIST_MAX_RETRIES=inf SCCACHE_SERVER_LOG=sccache=debug From 5b413544c6a9fda31911a2a864099cf1ebe2960e Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 31 Jul 2025 22:12:40 -0700 Subject: [PATCH 69/78] Revert "recache" This reverts commit b48f5846a05951b4222dc3ac15d4fedf7af169cb. --- .github/workflows/build-all-rapids-repos.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 2a4dc0057..ae0a5c075 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -46,7 +46,6 @@ jobs: # 3. Never fallback to locally compiling # 4. Use RAPIDS_AUX_SECRET_1 as the sccache-dist auth token env: | - SCCACHE_RECACHE=1 CONDA_ENV_CREATE_QUIET=1 SCCACHE_DIST_MAX_RETRIES=inf SCCACHE_SERVER_LOG=sccache=debug From 9971b288011bec035b1be0c2eb3bdb1c05eca7c4 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 5 Aug 2025 11:43:37 -0700 Subject: [PATCH 70/78] extract find_version_from_git_tags into separate file --- features/common/find-version-from-git-tags.sh | 109 ++++++++++++++++ features/common/utilities.sh | 121 +++++------------- .../common/find-version-from-git-tags.sh | 109 ++++++++++++++++ features/src/cccl-dev/common/utilities.sh | 121 +++++------------- .../common/find-version-from-git-tags.sh | 109 ++++++++++++++++ features/src/cmake/common/utilities.sh | 121 +++++------------- .../cuda/common/find-version-from-git-tags.sh | 109 ++++++++++++++++ features/src/cuda/common/utilities.sh | 121 +++++------------- .../gcc/common/find-version-from-git-tags.sh | 109 ++++++++++++++++ features/src/gcc/common/utilities.sh | 121 +++++------------- .../common/find-version-from-git-tags.sh | 109 ++++++++++++++++ features/src/gitlab-cli/common/utilities.sh | 121 +++++------------- .../llvm/common/find-version-from-git-tags.sh | 109 ++++++++++++++++ features/src/llvm/common/utilities.sh | 121 +++++------------- .../common/find-version-from-git-tags.sh | 109 ++++++++++++++++ features/src/mambaforge/common/utilities.sh | 121 +++++------------- .../common/find-version-from-git-tags.sh | 109 ++++++++++++++++ features/src/ninja/common/utilities.sh | 121 +++++------------- .../common/find-version-from-git-tags.sh | 109 ++++++++++++++++ features/src/nvhpc/common/utilities.sh | 121 +++++------------- .../common/find-version-from-git-tags.sh | 109 ++++++++++++++++ features/src/oneapi/common/utilities.sh | 121 +++++------------- .../common/find-version-from-git-tags.sh | 109 ++++++++++++++++ features/src/openmpi/common/utilities.sh | 121 +++++------------- .../common/find-version-from-git-tags.sh | 109 ++++++++++++++++ .../rapids-build-utils/common/utilities.sh | 121 +++++------------- .../rust/common/find-version-from-git-tags.sh | 109 ++++++++++++++++ features/src/rust/common/utilities.sh | 121 +++++------------- .../common/find-version-from-git-tags.sh | 109 ++++++++++++++++ features/src/sccache/common/utilities.sh | 121 +++++------------- .../ucx/common/find-version-from-git-tags.sh | 109 ++++++++++++++++ features/src/ucx/common/utilities.sh | 121 +++++------------- .../common/find-version-from-git-tags.sh | 109 ++++++++++++++++ features/src/utils/common/utilities.sh | 121 +++++------------- .../devcontainer/bin/post-attach-command.sh | 2 +- .../bin/sccache/find-version-from-git-tags.sh | 109 ++++++++++++++++ .../opt/devcontainer/bin/sccache/install.sh | 92 +++---------- scripts/copy-common-scripts.sh | 2 + 38 files changed, 2527 insertions(+), 1588 deletions(-) create mode 100644 features/common/find-version-from-git-tags.sh create mode 100644 features/src/cccl-dev/common/find-version-from-git-tags.sh create mode 100644 features/src/cmake/common/find-version-from-git-tags.sh create mode 100644 features/src/cuda/common/find-version-from-git-tags.sh create mode 100644 features/src/gcc/common/find-version-from-git-tags.sh create mode 100644 features/src/gitlab-cli/common/find-version-from-git-tags.sh create mode 100644 features/src/llvm/common/find-version-from-git-tags.sh create mode 100644 features/src/mambaforge/common/find-version-from-git-tags.sh create mode 100644 features/src/ninja/common/find-version-from-git-tags.sh create mode 100644 features/src/nvhpc/common/find-version-from-git-tags.sh create mode 100644 features/src/oneapi/common/find-version-from-git-tags.sh create mode 100644 features/src/openmpi/common/find-version-from-git-tags.sh create mode 100644 features/src/rapids-build-utils/common/find-version-from-git-tags.sh create mode 100644 features/src/rust/common/find-version-from-git-tags.sh create mode 100644 features/src/sccache/common/find-version-from-git-tags.sh create mode 100644 features/src/ucx/common/find-version-from-git-tags.sh create mode 100644 features/src/utils/common/find-version-from-git-tags.sh create mode 100644 features/src/utils/opt/devcontainer/bin/sccache/find-version-from-git-tags.sh diff --git a/features/common/find-version-from-git-tags.sh b/features/common/find-version-from-git-tags.sh new file mode 100644 index 000000000..9e9d649ce --- /dev/null +++ b/features/common/find-version-from-git-tags.sh @@ -0,0 +1,109 @@ +#! /usr/bin/env bash + +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + +# Figure out correct version of a three part version number is not passed +_find_version_from_git_tags() { + local variable_name="$1" + local requested_version="${!variable_name}" + if [ "${requested_version}" = "none" ]; then return; fi + local repository="$2" + local prefix="${3:-"tags/v"}" + local separator="${4:-"."}" + local suffix="${5:-}" + local last_part_optional="${6:-"false"}" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator="${separator//./\\.}" + local last_part="" + if [ "${last_part_optional}" = "true" ]; then + last_part+="(${escaped_separator}[0-9]+)?" + last_part+="(${escaped_separator}[0-9]+)?" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})?" + fi + else + last_part+="${escaped_separator}[0-9]+" + last_part+="${escaped_separator}[0-9]+" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})" + fi + fi + local regex="${prefix}\\K[0-9]+${last_part}$" + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch + fi + local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + fi + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + requested_version="$(head -n 1 <<< "${version_list}")" + else + set +e + requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" + set -e + fi + fi + if [ ! -n "${requested_version:+x}" ] || ! grep "^${requested_version//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + return 1 + fi + _upvar "${variable_name}" "${requested_version}" + echo "${variable_name}=${requested_version}" +} + +# Use semver logic to decrement a version number then look for the closest match +_find_prev_version_from_git_tags() { + local variable_name="$1" + local current_version="${!variable_name}" + local repository="$2" + # Normally a "v" is used before the version number, but support alternate cases + local prefix="${3:-"tags/v"}" + # Some repositories use "_" instead of "." for version number part separation, support that + local separator="${4:-"."}" + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex="${5:-}" + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional="${6:-"false"}" + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + local major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + local minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + local breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + _upvar "${variable_name}" "${major}" + # Look for latest version from previous major release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + _upvar "${variable_name}" "${major}.${minor}" + # Look for latest version from previous minor release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + _upvar "${variable_name}" "${major}.${minor}" + else + _upvar "${variable_name}" "${major}.${minor}.${breakfix}" + fi + fi + set -e +} diff --git a/features/common/utilities.sh b/features/common/utilities.sh index 89879e7bf..49a044b95 100644 --- a/features/common/utilities.sh +++ b/features/common/utilities.sh @@ -2,6 +2,22 @@ # Define common shell functions +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + # Run apt update if apt lists aren't populated apt_get_update() { if [ "$(find /var/lib/apt/lists -mindepth 1 | head -n1 | wc -l)" = "0" ]; then @@ -131,95 +147,6 @@ add_etc_profile_d_script() { export -f add_etc_profile_d_script; -# Figure out correct version of a three part version number is not passed -find_version_from_git_tags() { - check_packages git; - local variable_name=$1 - local requested_version=${!variable_name} - if [ "${requested_version}" = "none" ]; then return; fi - local repository=$2 - local prefix=${3:-"tags/v"} - local separator=${4:-"."} - local suffix=${5:-} - local last_part_optional=${6:-"false"} - if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then - local escaped_separator=${separator//./\\.} - local last_part= - if [ "${last_part_optional}" = "true" ]; then - last_part+="(${escaped_separator}[0-9]+)?" - last_part+="(${escaped_separator}[0-9]+)?" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})?" - fi - else - last_part+="${escaped_separator}[0-9]+" - last_part+="${escaped_separator}[0-9]+" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})" - fi - fi - local regex="${prefix}\\K[0-9]+${last_part}$" - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then - declare -g "${variable_name}"="$(echo "${version_list}" | head -n 1)" - else - set +e - declare -g "${variable_name}"="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" - set -e - fi - fi - if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then - echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 - exit 1 - fi - echo "${variable_name}=${!variable_name}" -} - -export -f find_version_from_git_tags; - -# Use semver logic to decrement a version number then look for the closest match -find_prev_version_from_git_tags() { - local variable_name=$1 - local current_version=${!variable_name} - local repository=$2 - # Normally a "v" is used before the version number, but support alternate cases - local prefix=${3:-"tags/v"} - # Some repositories use "_" instead of "." for version number part separation, support that - local separator=${4:-"."} - # Some repositories may have tags that include a suffix (e.g. actions/node-versions) - local version_suffix_regex=${5:-} - # Some tools release versions that omit the last digit (e.g. go) - local last_part_optional=${6:-"false"} - # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. - set +e - major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" - minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" - breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" - - if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then - ((major=major-1)) - declare -g "${variable_name}"="${major}" - # Look for latest version from previous major release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - # Handle situations like Go's odd version pattern where "0" releases omit the last part - elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then - ((minor=minor-1)) - declare -g "${variable_name}"="${major}.${minor}" - # Look for latest version from previous minor release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - else - ((breakfix=breakfix-1)) - if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then - declare -g "${variable_name}"="${major}.${minor}" - else - declare -g "${variable_name}"="${major}.${minor}.${breakfix}" - fi - fi - set -e -} - -export -f find_prev_version_from_git_tags; - # Determine the appropriate non-root user find_non_root_user() { USERNAME="${USERNAME:-"${_REMOTE_USER:-"auto"}"}"; @@ -241,3 +168,19 @@ find_non_root_user() { } export -f find_non_root_user; + +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/find-version-from-git-tags.sh"; + +find_version_from_git_tags() { + check_packages git; + _find_version_from_git_tags "$@"; +} + +export -f find_version_from_git_tags; + +find_prev_version_from_git_tags() { + _find_prev_version_from_git_tags "$@"; +} + +export -f find_prev_version_from_git_tags; diff --git a/features/src/cccl-dev/common/find-version-from-git-tags.sh b/features/src/cccl-dev/common/find-version-from-git-tags.sh new file mode 100644 index 000000000..9e9d649ce --- /dev/null +++ b/features/src/cccl-dev/common/find-version-from-git-tags.sh @@ -0,0 +1,109 @@ +#! /usr/bin/env bash + +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + +# Figure out correct version of a three part version number is not passed +_find_version_from_git_tags() { + local variable_name="$1" + local requested_version="${!variable_name}" + if [ "${requested_version}" = "none" ]; then return; fi + local repository="$2" + local prefix="${3:-"tags/v"}" + local separator="${4:-"."}" + local suffix="${5:-}" + local last_part_optional="${6:-"false"}" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator="${separator//./\\.}" + local last_part="" + if [ "${last_part_optional}" = "true" ]; then + last_part+="(${escaped_separator}[0-9]+)?" + last_part+="(${escaped_separator}[0-9]+)?" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})?" + fi + else + last_part+="${escaped_separator}[0-9]+" + last_part+="${escaped_separator}[0-9]+" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})" + fi + fi + local regex="${prefix}\\K[0-9]+${last_part}$" + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch + fi + local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + fi + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + requested_version="$(head -n 1 <<< "${version_list}")" + else + set +e + requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" + set -e + fi + fi + if [ ! -n "${requested_version:+x}" ] || ! grep "^${requested_version//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + return 1 + fi + _upvar "${variable_name}" "${requested_version}" + echo "${variable_name}=${requested_version}" +} + +# Use semver logic to decrement a version number then look for the closest match +_find_prev_version_from_git_tags() { + local variable_name="$1" + local current_version="${!variable_name}" + local repository="$2" + # Normally a "v" is used before the version number, but support alternate cases + local prefix="${3:-"tags/v"}" + # Some repositories use "_" instead of "." for version number part separation, support that + local separator="${4:-"."}" + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex="${5:-}" + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional="${6:-"false"}" + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + local major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + local minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + local breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + _upvar "${variable_name}" "${major}" + # Look for latest version from previous major release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + _upvar "${variable_name}" "${major}.${minor}" + # Look for latest version from previous minor release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + _upvar "${variable_name}" "${major}.${minor}" + else + _upvar "${variable_name}" "${major}.${minor}.${breakfix}" + fi + fi + set -e +} diff --git a/features/src/cccl-dev/common/utilities.sh b/features/src/cccl-dev/common/utilities.sh index 89879e7bf..49a044b95 100644 --- a/features/src/cccl-dev/common/utilities.sh +++ b/features/src/cccl-dev/common/utilities.sh @@ -2,6 +2,22 @@ # Define common shell functions +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + # Run apt update if apt lists aren't populated apt_get_update() { if [ "$(find /var/lib/apt/lists -mindepth 1 | head -n1 | wc -l)" = "0" ]; then @@ -131,95 +147,6 @@ add_etc_profile_d_script() { export -f add_etc_profile_d_script; -# Figure out correct version of a three part version number is not passed -find_version_from_git_tags() { - check_packages git; - local variable_name=$1 - local requested_version=${!variable_name} - if [ "${requested_version}" = "none" ]; then return; fi - local repository=$2 - local prefix=${3:-"tags/v"} - local separator=${4:-"."} - local suffix=${5:-} - local last_part_optional=${6:-"false"} - if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then - local escaped_separator=${separator//./\\.} - local last_part= - if [ "${last_part_optional}" = "true" ]; then - last_part+="(${escaped_separator}[0-9]+)?" - last_part+="(${escaped_separator}[0-9]+)?" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})?" - fi - else - last_part+="${escaped_separator}[0-9]+" - last_part+="${escaped_separator}[0-9]+" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})" - fi - fi - local regex="${prefix}\\K[0-9]+${last_part}$" - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then - declare -g "${variable_name}"="$(echo "${version_list}" | head -n 1)" - else - set +e - declare -g "${variable_name}"="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" - set -e - fi - fi - if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then - echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 - exit 1 - fi - echo "${variable_name}=${!variable_name}" -} - -export -f find_version_from_git_tags; - -# Use semver logic to decrement a version number then look for the closest match -find_prev_version_from_git_tags() { - local variable_name=$1 - local current_version=${!variable_name} - local repository=$2 - # Normally a "v" is used before the version number, but support alternate cases - local prefix=${3:-"tags/v"} - # Some repositories use "_" instead of "." for version number part separation, support that - local separator=${4:-"."} - # Some repositories may have tags that include a suffix (e.g. actions/node-versions) - local version_suffix_regex=${5:-} - # Some tools release versions that omit the last digit (e.g. go) - local last_part_optional=${6:-"false"} - # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. - set +e - major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" - minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" - breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" - - if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then - ((major=major-1)) - declare -g "${variable_name}"="${major}" - # Look for latest version from previous major release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - # Handle situations like Go's odd version pattern where "0" releases omit the last part - elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then - ((minor=minor-1)) - declare -g "${variable_name}"="${major}.${minor}" - # Look for latest version from previous minor release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - else - ((breakfix=breakfix-1)) - if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then - declare -g "${variable_name}"="${major}.${minor}" - else - declare -g "${variable_name}"="${major}.${minor}.${breakfix}" - fi - fi - set -e -} - -export -f find_prev_version_from_git_tags; - # Determine the appropriate non-root user find_non_root_user() { USERNAME="${USERNAME:-"${_REMOTE_USER:-"auto"}"}"; @@ -241,3 +168,19 @@ find_non_root_user() { } export -f find_non_root_user; + +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/find-version-from-git-tags.sh"; + +find_version_from_git_tags() { + check_packages git; + _find_version_from_git_tags "$@"; +} + +export -f find_version_from_git_tags; + +find_prev_version_from_git_tags() { + _find_prev_version_from_git_tags "$@"; +} + +export -f find_prev_version_from_git_tags; diff --git a/features/src/cmake/common/find-version-from-git-tags.sh b/features/src/cmake/common/find-version-from-git-tags.sh new file mode 100644 index 000000000..9e9d649ce --- /dev/null +++ b/features/src/cmake/common/find-version-from-git-tags.sh @@ -0,0 +1,109 @@ +#! /usr/bin/env bash + +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + +# Figure out correct version of a three part version number is not passed +_find_version_from_git_tags() { + local variable_name="$1" + local requested_version="${!variable_name}" + if [ "${requested_version}" = "none" ]; then return; fi + local repository="$2" + local prefix="${3:-"tags/v"}" + local separator="${4:-"."}" + local suffix="${5:-}" + local last_part_optional="${6:-"false"}" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator="${separator//./\\.}" + local last_part="" + if [ "${last_part_optional}" = "true" ]; then + last_part+="(${escaped_separator}[0-9]+)?" + last_part+="(${escaped_separator}[0-9]+)?" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})?" + fi + else + last_part+="${escaped_separator}[0-9]+" + last_part+="${escaped_separator}[0-9]+" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})" + fi + fi + local regex="${prefix}\\K[0-9]+${last_part}$" + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch + fi + local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + fi + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + requested_version="$(head -n 1 <<< "${version_list}")" + else + set +e + requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" + set -e + fi + fi + if [ ! -n "${requested_version:+x}" ] || ! grep "^${requested_version//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + return 1 + fi + _upvar "${variable_name}" "${requested_version}" + echo "${variable_name}=${requested_version}" +} + +# Use semver logic to decrement a version number then look for the closest match +_find_prev_version_from_git_tags() { + local variable_name="$1" + local current_version="${!variable_name}" + local repository="$2" + # Normally a "v" is used before the version number, but support alternate cases + local prefix="${3:-"tags/v"}" + # Some repositories use "_" instead of "." for version number part separation, support that + local separator="${4:-"."}" + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex="${5:-}" + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional="${6:-"false"}" + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + local major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + local minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + local breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + _upvar "${variable_name}" "${major}" + # Look for latest version from previous major release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + _upvar "${variable_name}" "${major}.${minor}" + # Look for latest version from previous minor release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + _upvar "${variable_name}" "${major}.${minor}" + else + _upvar "${variable_name}" "${major}.${minor}.${breakfix}" + fi + fi + set -e +} diff --git a/features/src/cmake/common/utilities.sh b/features/src/cmake/common/utilities.sh index 89879e7bf..49a044b95 100644 --- a/features/src/cmake/common/utilities.sh +++ b/features/src/cmake/common/utilities.sh @@ -2,6 +2,22 @@ # Define common shell functions +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + # Run apt update if apt lists aren't populated apt_get_update() { if [ "$(find /var/lib/apt/lists -mindepth 1 | head -n1 | wc -l)" = "0" ]; then @@ -131,95 +147,6 @@ add_etc_profile_d_script() { export -f add_etc_profile_d_script; -# Figure out correct version of a three part version number is not passed -find_version_from_git_tags() { - check_packages git; - local variable_name=$1 - local requested_version=${!variable_name} - if [ "${requested_version}" = "none" ]; then return; fi - local repository=$2 - local prefix=${3:-"tags/v"} - local separator=${4:-"."} - local suffix=${5:-} - local last_part_optional=${6:-"false"} - if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then - local escaped_separator=${separator//./\\.} - local last_part= - if [ "${last_part_optional}" = "true" ]; then - last_part+="(${escaped_separator}[0-9]+)?" - last_part+="(${escaped_separator}[0-9]+)?" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})?" - fi - else - last_part+="${escaped_separator}[0-9]+" - last_part+="${escaped_separator}[0-9]+" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})" - fi - fi - local regex="${prefix}\\K[0-9]+${last_part}$" - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then - declare -g "${variable_name}"="$(echo "${version_list}" | head -n 1)" - else - set +e - declare -g "${variable_name}"="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" - set -e - fi - fi - if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then - echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 - exit 1 - fi - echo "${variable_name}=${!variable_name}" -} - -export -f find_version_from_git_tags; - -# Use semver logic to decrement a version number then look for the closest match -find_prev_version_from_git_tags() { - local variable_name=$1 - local current_version=${!variable_name} - local repository=$2 - # Normally a "v" is used before the version number, but support alternate cases - local prefix=${3:-"tags/v"} - # Some repositories use "_" instead of "." for version number part separation, support that - local separator=${4:-"."} - # Some repositories may have tags that include a suffix (e.g. actions/node-versions) - local version_suffix_regex=${5:-} - # Some tools release versions that omit the last digit (e.g. go) - local last_part_optional=${6:-"false"} - # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. - set +e - major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" - minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" - breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" - - if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then - ((major=major-1)) - declare -g "${variable_name}"="${major}" - # Look for latest version from previous major release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - # Handle situations like Go's odd version pattern where "0" releases omit the last part - elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then - ((minor=minor-1)) - declare -g "${variable_name}"="${major}.${minor}" - # Look for latest version from previous minor release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - else - ((breakfix=breakfix-1)) - if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then - declare -g "${variable_name}"="${major}.${minor}" - else - declare -g "${variable_name}"="${major}.${minor}.${breakfix}" - fi - fi - set -e -} - -export -f find_prev_version_from_git_tags; - # Determine the appropriate non-root user find_non_root_user() { USERNAME="${USERNAME:-"${_REMOTE_USER:-"auto"}"}"; @@ -241,3 +168,19 @@ find_non_root_user() { } export -f find_non_root_user; + +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/find-version-from-git-tags.sh"; + +find_version_from_git_tags() { + check_packages git; + _find_version_from_git_tags "$@"; +} + +export -f find_version_from_git_tags; + +find_prev_version_from_git_tags() { + _find_prev_version_from_git_tags "$@"; +} + +export -f find_prev_version_from_git_tags; diff --git a/features/src/cuda/common/find-version-from-git-tags.sh b/features/src/cuda/common/find-version-from-git-tags.sh new file mode 100644 index 000000000..9e9d649ce --- /dev/null +++ b/features/src/cuda/common/find-version-from-git-tags.sh @@ -0,0 +1,109 @@ +#! /usr/bin/env bash + +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + +# Figure out correct version of a three part version number is not passed +_find_version_from_git_tags() { + local variable_name="$1" + local requested_version="${!variable_name}" + if [ "${requested_version}" = "none" ]; then return; fi + local repository="$2" + local prefix="${3:-"tags/v"}" + local separator="${4:-"."}" + local suffix="${5:-}" + local last_part_optional="${6:-"false"}" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator="${separator//./\\.}" + local last_part="" + if [ "${last_part_optional}" = "true" ]; then + last_part+="(${escaped_separator}[0-9]+)?" + last_part+="(${escaped_separator}[0-9]+)?" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})?" + fi + else + last_part+="${escaped_separator}[0-9]+" + last_part+="${escaped_separator}[0-9]+" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})" + fi + fi + local regex="${prefix}\\K[0-9]+${last_part}$" + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch + fi + local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + fi + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + requested_version="$(head -n 1 <<< "${version_list}")" + else + set +e + requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" + set -e + fi + fi + if [ ! -n "${requested_version:+x}" ] || ! grep "^${requested_version//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + return 1 + fi + _upvar "${variable_name}" "${requested_version}" + echo "${variable_name}=${requested_version}" +} + +# Use semver logic to decrement a version number then look for the closest match +_find_prev_version_from_git_tags() { + local variable_name="$1" + local current_version="${!variable_name}" + local repository="$2" + # Normally a "v" is used before the version number, but support alternate cases + local prefix="${3:-"tags/v"}" + # Some repositories use "_" instead of "." for version number part separation, support that + local separator="${4:-"."}" + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex="${5:-}" + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional="${6:-"false"}" + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + local major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + local minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + local breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + _upvar "${variable_name}" "${major}" + # Look for latest version from previous major release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + _upvar "${variable_name}" "${major}.${minor}" + # Look for latest version from previous minor release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + _upvar "${variable_name}" "${major}.${minor}" + else + _upvar "${variable_name}" "${major}.${minor}.${breakfix}" + fi + fi + set -e +} diff --git a/features/src/cuda/common/utilities.sh b/features/src/cuda/common/utilities.sh index 89879e7bf..49a044b95 100644 --- a/features/src/cuda/common/utilities.sh +++ b/features/src/cuda/common/utilities.sh @@ -2,6 +2,22 @@ # Define common shell functions +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + # Run apt update if apt lists aren't populated apt_get_update() { if [ "$(find /var/lib/apt/lists -mindepth 1 | head -n1 | wc -l)" = "0" ]; then @@ -131,95 +147,6 @@ add_etc_profile_d_script() { export -f add_etc_profile_d_script; -# Figure out correct version of a three part version number is not passed -find_version_from_git_tags() { - check_packages git; - local variable_name=$1 - local requested_version=${!variable_name} - if [ "${requested_version}" = "none" ]; then return; fi - local repository=$2 - local prefix=${3:-"tags/v"} - local separator=${4:-"."} - local suffix=${5:-} - local last_part_optional=${6:-"false"} - if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then - local escaped_separator=${separator//./\\.} - local last_part= - if [ "${last_part_optional}" = "true" ]; then - last_part+="(${escaped_separator}[0-9]+)?" - last_part+="(${escaped_separator}[0-9]+)?" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})?" - fi - else - last_part+="${escaped_separator}[0-9]+" - last_part+="${escaped_separator}[0-9]+" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})" - fi - fi - local regex="${prefix}\\K[0-9]+${last_part}$" - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then - declare -g "${variable_name}"="$(echo "${version_list}" | head -n 1)" - else - set +e - declare -g "${variable_name}"="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" - set -e - fi - fi - if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then - echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 - exit 1 - fi - echo "${variable_name}=${!variable_name}" -} - -export -f find_version_from_git_tags; - -# Use semver logic to decrement a version number then look for the closest match -find_prev_version_from_git_tags() { - local variable_name=$1 - local current_version=${!variable_name} - local repository=$2 - # Normally a "v" is used before the version number, but support alternate cases - local prefix=${3:-"tags/v"} - # Some repositories use "_" instead of "." for version number part separation, support that - local separator=${4:-"."} - # Some repositories may have tags that include a suffix (e.g. actions/node-versions) - local version_suffix_regex=${5:-} - # Some tools release versions that omit the last digit (e.g. go) - local last_part_optional=${6:-"false"} - # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. - set +e - major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" - minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" - breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" - - if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then - ((major=major-1)) - declare -g "${variable_name}"="${major}" - # Look for latest version from previous major release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - # Handle situations like Go's odd version pattern where "0" releases omit the last part - elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then - ((minor=minor-1)) - declare -g "${variable_name}"="${major}.${minor}" - # Look for latest version from previous minor release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - else - ((breakfix=breakfix-1)) - if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then - declare -g "${variable_name}"="${major}.${minor}" - else - declare -g "${variable_name}"="${major}.${minor}.${breakfix}" - fi - fi - set -e -} - -export -f find_prev_version_from_git_tags; - # Determine the appropriate non-root user find_non_root_user() { USERNAME="${USERNAME:-"${_REMOTE_USER:-"auto"}"}"; @@ -241,3 +168,19 @@ find_non_root_user() { } export -f find_non_root_user; + +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/find-version-from-git-tags.sh"; + +find_version_from_git_tags() { + check_packages git; + _find_version_from_git_tags "$@"; +} + +export -f find_version_from_git_tags; + +find_prev_version_from_git_tags() { + _find_prev_version_from_git_tags "$@"; +} + +export -f find_prev_version_from_git_tags; diff --git a/features/src/gcc/common/find-version-from-git-tags.sh b/features/src/gcc/common/find-version-from-git-tags.sh new file mode 100644 index 000000000..9e9d649ce --- /dev/null +++ b/features/src/gcc/common/find-version-from-git-tags.sh @@ -0,0 +1,109 @@ +#! /usr/bin/env bash + +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + +# Figure out correct version of a three part version number is not passed +_find_version_from_git_tags() { + local variable_name="$1" + local requested_version="${!variable_name}" + if [ "${requested_version}" = "none" ]; then return; fi + local repository="$2" + local prefix="${3:-"tags/v"}" + local separator="${4:-"."}" + local suffix="${5:-}" + local last_part_optional="${6:-"false"}" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator="${separator//./\\.}" + local last_part="" + if [ "${last_part_optional}" = "true" ]; then + last_part+="(${escaped_separator}[0-9]+)?" + last_part+="(${escaped_separator}[0-9]+)?" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})?" + fi + else + last_part+="${escaped_separator}[0-9]+" + last_part+="${escaped_separator}[0-9]+" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})" + fi + fi + local regex="${prefix}\\K[0-9]+${last_part}$" + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch + fi + local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + fi + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + requested_version="$(head -n 1 <<< "${version_list}")" + else + set +e + requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" + set -e + fi + fi + if [ ! -n "${requested_version:+x}" ] || ! grep "^${requested_version//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + return 1 + fi + _upvar "${variable_name}" "${requested_version}" + echo "${variable_name}=${requested_version}" +} + +# Use semver logic to decrement a version number then look for the closest match +_find_prev_version_from_git_tags() { + local variable_name="$1" + local current_version="${!variable_name}" + local repository="$2" + # Normally a "v" is used before the version number, but support alternate cases + local prefix="${3:-"tags/v"}" + # Some repositories use "_" instead of "." for version number part separation, support that + local separator="${4:-"."}" + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex="${5:-}" + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional="${6:-"false"}" + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + local major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + local minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + local breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + _upvar "${variable_name}" "${major}" + # Look for latest version from previous major release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + _upvar "${variable_name}" "${major}.${minor}" + # Look for latest version from previous minor release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + _upvar "${variable_name}" "${major}.${minor}" + else + _upvar "${variable_name}" "${major}.${minor}.${breakfix}" + fi + fi + set -e +} diff --git a/features/src/gcc/common/utilities.sh b/features/src/gcc/common/utilities.sh index 89879e7bf..49a044b95 100644 --- a/features/src/gcc/common/utilities.sh +++ b/features/src/gcc/common/utilities.sh @@ -2,6 +2,22 @@ # Define common shell functions +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + # Run apt update if apt lists aren't populated apt_get_update() { if [ "$(find /var/lib/apt/lists -mindepth 1 | head -n1 | wc -l)" = "0" ]; then @@ -131,95 +147,6 @@ add_etc_profile_d_script() { export -f add_etc_profile_d_script; -# Figure out correct version of a three part version number is not passed -find_version_from_git_tags() { - check_packages git; - local variable_name=$1 - local requested_version=${!variable_name} - if [ "${requested_version}" = "none" ]; then return; fi - local repository=$2 - local prefix=${3:-"tags/v"} - local separator=${4:-"."} - local suffix=${5:-} - local last_part_optional=${6:-"false"} - if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then - local escaped_separator=${separator//./\\.} - local last_part= - if [ "${last_part_optional}" = "true" ]; then - last_part+="(${escaped_separator}[0-9]+)?" - last_part+="(${escaped_separator}[0-9]+)?" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})?" - fi - else - last_part+="${escaped_separator}[0-9]+" - last_part+="${escaped_separator}[0-9]+" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})" - fi - fi - local regex="${prefix}\\K[0-9]+${last_part}$" - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then - declare -g "${variable_name}"="$(echo "${version_list}" | head -n 1)" - else - set +e - declare -g "${variable_name}"="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" - set -e - fi - fi - if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then - echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 - exit 1 - fi - echo "${variable_name}=${!variable_name}" -} - -export -f find_version_from_git_tags; - -# Use semver logic to decrement a version number then look for the closest match -find_prev_version_from_git_tags() { - local variable_name=$1 - local current_version=${!variable_name} - local repository=$2 - # Normally a "v" is used before the version number, but support alternate cases - local prefix=${3:-"tags/v"} - # Some repositories use "_" instead of "." for version number part separation, support that - local separator=${4:-"."} - # Some repositories may have tags that include a suffix (e.g. actions/node-versions) - local version_suffix_regex=${5:-} - # Some tools release versions that omit the last digit (e.g. go) - local last_part_optional=${6:-"false"} - # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. - set +e - major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" - minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" - breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" - - if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then - ((major=major-1)) - declare -g "${variable_name}"="${major}" - # Look for latest version from previous major release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - # Handle situations like Go's odd version pattern where "0" releases omit the last part - elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then - ((minor=minor-1)) - declare -g "${variable_name}"="${major}.${minor}" - # Look for latest version from previous minor release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - else - ((breakfix=breakfix-1)) - if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then - declare -g "${variable_name}"="${major}.${minor}" - else - declare -g "${variable_name}"="${major}.${minor}.${breakfix}" - fi - fi - set -e -} - -export -f find_prev_version_from_git_tags; - # Determine the appropriate non-root user find_non_root_user() { USERNAME="${USERNAME:-"${_REMOTE_USER:-"auto"}"}"; @@ -241,3 +168,19 @@ find_non_root_user() { } export -f find_non_root_user; + +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/find-version-from-git-tags.sh"; + +find_version_from_git_tags() { + check_packages git; + _find_version_from_git_tags "$@"; +} + +export -f find_version_from_git_tags; + +find_prev_version_from_git_tags() { + _find_prev_version_from_git_tags "$@"; +} + +export -f find_prev_version_from_git_tags; diff --git a/features/src/gitlab-cli/common/find-version-from-git-tags.sh b/features/src/gitlab-cli/common/find-version-from-git-tags.sh new file mode 100644 index 000000000..9e9d649ce --- /dev/null +++ b/features/src/gitlab-cli/common/find-version-from-git-tags.sh @@ -0,0 +1,109 @@ +#! /usr/bin/env bash + +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + +# Figure out correct version of a three part version number is not passed +_find_version_from_git_tags() { + local variable_name="$1" + local requested_version="${!variable_name}" + if [ "${requested_version}" = "none" ]; then return; fi + local repository="$2" + local prefix="${3:-"tags/v"}" + local separator="${4:-"."}" + local suffix="${5:-}" + local last_part_optional="${6:-"false"}" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator="${separator//./\\.}" + local last_part="" + if [ "${last_part_optional}" = "true" ]; then + last_part+="(${escaped_separator}[0-9]+)?" + last_part+="(${escaped_separator}[0-9]+)?" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})?" + fi + else + last_part+="${escaped_separator}[0-9]+" + last_part+="${escaped_separator}[0-9]+" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})" + fi + fi + local regex="${prefix}\\K[0-9]+${last_part}$" + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch + fi + local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + fi + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + requested_version="$(head -n 1 <<< "${version_list}")" + else + set +e + requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" + set -e + fi + fi + if [ ! -n "${requested_version:+x}" ] || ! grep "^${requested_version//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + return 1 + fi + _upvar "${variable_name}" "${requested_version}" + echo "${variable_name}=${requested_version}" +} + +# Use semver logic to decrement a version number then look for the closest match +_find_prev_version_from_git_tags() { + local variable_name="$1" + local current_version="${!variable_name}" + local repository="$2" + # Normally a "v" is used before the version number, but support alternate cases + local prefix="${3:-"tags/v"}" + # Some repositories use "_" instead of "." for version number part separation, support that + local separator="${4:-"."}" + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex="${5:-}" + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional="${6:-"false"}" + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + local major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + local minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + local breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + _upvar "${variable_name}" "${major}" + # Look for latest version from previous major release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + _upvar "${variable_name}" "${major}.${minor}" + # Look for latest version from previous minor release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + _upvar "${variable_name}" "${major}.${minor}" + else + _upvar "${variable_name}" "${major}.${minor}.${breakfix}" + fi + fi + set -e +} diff --git a/features/src/gitlab-cli/common/utilities.sh b/features/src/gitlab-cli/common/utilities.sh index 89879e7bf..49a044b95 100644 --- a/features/src/gitlab-cli/common/utilities.sh +++ b/features/src/gitlab-cli/common/utilities.sh @@ -2,6 +2,22 @@ # Define common shell functions +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + # Run apt update if apt lists aren't populated apt_get_update() { if [ "$(find /var/lib/apt/lists -mindepth 1 | head -n1 | wc -l)" = "0" ]; then @@ -131,95 +147,6 @@ add_etc_profile_d_script() { export -f add_etc_profile_d_script; -# Figure out correct version of a three part version number is not passed -find_version_from_git_tags() { - check_packages git; - local variable_name=$1 - local requested_version=${!variable_name} - if [ "${requested_version}" = "none" ]; then return; fi - local repository=$2 - local prefix=${3:-"tags/v"} - local separator=${4:-"."} - local suffix=${5:-} - local last_part_optional=${6:-"false"} - if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then - local escaped_separator=${separator//./\\.} - local last_part= - if [ "${last_part_optional}" = "true" ]; then - last_part+="(${escaped_separator}[0-9]+)?" - last_part+="(${escaped_separator}[0-9]+)?" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})?" - fi - else - last_part+="${escaped_separator}[0-9]+" - last_part+="${escaped_separator}[0-9]+" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})" - fi - fi - local regex="${prefix}\\K[0-9]+${last_part}$" - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then - declare -g "${variable_name}"="$(echo "${version_list}" | head -n 1)" - else - set +e - declare -g "${variable_name}"="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" - set -e - fi - fi - if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then - echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 - exit 1 - fi - echo "${variable_name}=${!variable_name}" -} - -export -f find_version_from_git_tags; - -# Use semver logic to decrement a version number then look for the closest match -find_prev_version_from_git_tags() { - local variable_name=$1 - local current_version=${!variable_name} - local repository=$2 - # Normally a "v" is used before the version number, but support alternate cases - local prefix=${3:-"tags/v"} - # Some repositories use "_" instead of "." for version number part separation, support that - local separator=${4:-"."} - # Some repositories may have tags that include a suffix (e.g. actions/node-versions) - local version_suffix_regex=${5:-} - # Some tools release versions that omit the last digit (e.g. go) - local last_part_optional=${6:-"false"} - # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. - set +e - major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" - minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" - breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" - - if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then - ((major=major-1)) - declare -g "${variable_name}"="${major}" - # Look for latest version from previous major release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - # Handle situations like Go's odd version pattern where "0" releases omit the last part - elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then - ((minor=minor-1)) - declare -g "${variable_name}"="${major}.${minor}" - # Look for latest version from previous minor release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - else - ((breakfix=breakfix-1)) - if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then - declare -g "${variable_name}"="${major}.${minor}" - else - declare -g "${variable_name}"="${major}.${minor}.${breakfix}" - fi - fi - set -e -} - -export -f find_prev_version_from_git_tags; - # Determine the appropriate non-root user find_non_root_user() { USERNAME="${USERNAME:-"${_REMOTE_USER:-"auto"}"}"; @@ -241,3 +168,19 @@ find_non_root_user() { } export -f find_non_root_user; + +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/find-version-from-git-tags.sh"; + +find_version_from_git_tags() { + check_packages git; + _find_version_from_git_tags "$@"; +} + +export -f find_version_from_git_tags; + +find_prev_version_from_git_tags() { + _find_prev_version_from_git_tags "$@"; +} + +export -f find_prev_version_from_git_tags; diff --git a/features/src/llvm/common/find-version-from-git-tags.sh b/features/src/llvm/common/find-version-from-git-tags.sh new file mode 100644 index 000000000..9e9d649ce --- /dev/null +++ b/features/src/llvm/common/find-version-from-git-tags.sh @@ -0,0 +1,109 @@ +#! /usr/bin/env bash + +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + +# Figure out correct version of a three part version number is not passed +_find_version_from_git_tags() { + local variable_name="$1" + local requested_version="${!variable_name}" + if [ "${requested_version}" = "none" ]; then return; fi + local repository="$2" + local prefix="${3:-"tags/v"}" + local separator="${4:-"."}" + local suffix="${5:-}" + local last_part_optional="${6:-"false"}" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator="${separator//./\\.}" + local last_part="" + if [ "${last_part_optional}" = "true" ]; then + last_part+="(${escaped_separator}[0-9]+)?" + last_part+="(${escaped_separator}[0-9]+)?" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})?" + fi + else + last_part+="${escaped_separator}[0-9]+" + last_part+="${escaped_separator}[0-9]+" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})" + fi + fi + local regex="${prefix}\\K[0-9]+${last_part}$" + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch + fi + local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + fi + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + requested_version="$(head -n 1 <<< "${version_list}")" + else + set +e + requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" + set -e + fi + fi + if [ ! -n "${requested_version:+x}" ] || ! grep "^${requested_version//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + return 1 + fi + _upvar "${variable_name}" "${requested_version}" + echo "${variable_name}=${requested_version}" +} + +# Use semver logic to decrement a version number then look for the closest match +_find_prev_version_from_git_tags() { + local variable_name="$1" + local current_version="${!variable_name}" + local repository="$2" + # Normally a "v" is used before the version number, but support alternate cases + local prefix="${3:-"tags/v"}" + # Some repositories use "_" instead of "." for version number part separation, support that + local separator="${4:-"."}" + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex="${5:-}" + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional="${6:-"false"}" + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + local major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + local minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + local breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + _upvar "${variable_name}" "${major}" + # Look for latest version from previous major release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + _upvar "${variable_name}" "${major}.${minor}" + # Look for latest version from previous minor release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + _upvar "${variable_name}" "${major}.${minor}" + else + _upvar "${variable_name}" "${major}.${minor}.${breakfix}" + fi + fi + set -e +} diff --git a/features/src/llvm/common/utilities.sh b/features/src/llvm/common/utilities.sh index 89879e7bf..49a044b95 100644 --- a/features/src/llvm/common/utilities.sh +++ b/features/src/llvm/common/utilities.sh @@ -2,6 +2,22 @@ # Define common shell functions +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + # Run apt update if apt lists aren't populated apt_get_update() { if [ "$(find /var/lib/apt/lists -mindepth 1 | head -n1 | wc -l)" = "0" ]; then @@ -131,95 +147,6 @@ add_etc_profile_d_script() { export -f add_etc_profile_d_script; -# Figure out correct version of a three part version number is not passed -find_version_from_git_tags() { - check_packages git; - local variable_name=$1 - local requested_version=${!variable_name} - if [ "${requested_version}" = "none" ]; then return; fi - local repository=$2 - local prefix=${3:-"tags/v"} - local separator=${4:-"."} - local suffix=${5:-} - local last_part_optional=${6:-"false"} - if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then - local escaped_separator=${separator//./\\.} - local last_part= - if [ "${last_part_optional}" = "true" ]; then - last_part+="(${escaped_separator}[0-9]+)?" - last_part+="(${escaped_separator}[0-9]+)?" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})?" - fi - else - last_part+="${escaped_separator}[0-9]+" - last_part+="${escaped_separator}[0-9]+" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})" - fi - fi - local regex="${prefix}\\K[0-9]+${last_part}$" - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then - declare -g "${variable_name}"="$(echo "${version_list}" | head -n 1)" - else - set +e - declare -g "${variable_name}"="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" - set -e - fi - fi - if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then - echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 - exit 1 - fi - echo "${variable_name}=${!variable_name}" -} - -export -f find_version_from_git_tags; - -# Use semver logic to decrement a version number then look for the closest match -find_prev_version_from_git_tags() { - local variable_name=$1 - local current_version=${!variable_name} - local repository=$2 - # Normally a "v" is used before the version number, but support alternate cases - local prefix=${3:-"tags/v"} - # Some repositories use "_" instead of "." for version number part separation, support that - local separator=${4:-"."} - # Some repositories may have tags that include a suffix (e.g. actions/node-versions) - local version_suffix_regex=${5:-} - # Some tools release versions that omit the last digit (e.g. go) - local last_part_optional=${6:-"false"} - # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. - set +e - major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" - minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" - breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" - - if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then - ((major=major-1)) - declare -g "${variable_name}"="${major}" - # Look for latest version from previous major release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - # Handle situations like Go's odd version pattern where "0" releases omit the last part - elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then - ((minor=minor-1)) - declare -g "${variable_name}"="${major}.${minor}" - # Look for latest version from previous minor release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - else - ((breakfix=breakfix-1)) - if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then - declare -g "${variable_name}"="${major}.${minor}" - else - declare -g "${variable_name}"="${major}.${minor}.${breakfix}" - fi - fi - set -e -} - -export -f find_prev_version_from_git_tags; - # Determine the appropriate non-root user find_non_root_user() { USERNAME="${USERNAME:-"${_REMOTE_USER:-"auto"}"}"; @@ -241,3 +168,19 @@ find_non_root_user() { } export -f find_non_root_user; + +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/find-version-from-git-tags.sh"; + +find_version_from_git_tags() { + check_packages git; + _find_version_from_git_tags "$@"; +} + +export -f find_version_from_git_tags; + +find_prev_version_from_git_tags() { + _find_prev_version_from_git_tags "$@"; +} + +export -f find_prev_version_from_git_tags; diff --git a/features/src/mambaforge/common/find-version-from-git-tags.sh b/features/src/mambaforge/common/find-version-from-git-tags.sh new file mode 100644 index 000000000..9e9d649ce --- /dev/null +++ b/features/src/mambaforge/common/find-version-from-git-tags.sh @@ -0,0 +1,109 @@ +#! /usr/bin/env bash + +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + +# Figure out correct version of a three part version number is not passed +_find_version_from_git_tags() { + local variable_name="$1" + local requested_version="${!variable_name}" + if [ "${requested_version}" = "none" ]; then return; fi + local repository="$2" + local prefix="${3:-"tags/v"}" + local separator="${4:-"."}" + local suffix="${5:-}" + local last_part_optional="${6:-"false"}" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator="${separator//./\\.}" + local last_part="" + if [ "${last_part_optional}" = "true" ]; then + last_part+="(${escaped_separator}[0-9]+)?" + last_part+="(${escaped_separator}[0-9]+)?" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})?" + fi + else + last_part+="${escaped_separator}[0-9]+" + last_part+="${escaped_separator}[0-9]+" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})" + fi + fi + local regex="${prefix}\\K[0-9]+${last_part}$" + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch + fi + local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + fi + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + requested_version="$(head -n 1 <<< "${version_list}")" + else + set +e + requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" + set -e + fi + fi + if [ ! -n "${requested_version:+x}" ] || ! grep "^${requested_version//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + return 1 + fi + _upvar "${variable_name}" "${requested_version}" + echo "${variable_name}=${requested_version}" +} + +# Use semver logic to decrement a version number then look for the closest match +_find_prev_version_from_git_tags() { + local variable_name="$1" + local current_version="${!variable_name}" + local repository="$2" + # Normally a "v" is used before the version number, but support alternate cases + local prefix="${3:-"tags/v"}" + # Some repositories use "_" instead of "." for version number part separation, support that + local separator="${4:-"."}" + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex="${5:-}" + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional="${6:-"false"}" + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + local major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + local minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + local breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + _upvar "${variable_name}" "${major}" + # Look for latest version from previous major release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + _upvar "${variable_name}" "${major}.${minor}" + # Look for latest version from previous minor release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + _upvar "${variable_name}" "${major}.${minor}" + else + _upvar "${variable_name}" "${major}.${minor}.${breakfix}" + fi + fi + set -e +} diff --git a/features/src/mambaforge/common/utilities.sh b/features/src/mambaforge/common/utilities.sh index 89879e7bf..49a044b95 100644 --- a/features/src/mambaforge/common/utilities.sh +++ b/features/src/mambaforge/common/utilities.sh @@ -2,6 +2,22 @@ # Define common shell functions +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + # Run apt update if apt lists aren't populated apt_get_update() { if [ "$(find /var/lib/apt/lists -mindepth 1 | head -n1 | wc -l)" = "0" ]; then @@ -131,95 +147,6 @@ add_etc_profile_d_script() { export -f add_etc_profile_d_script; -# Figure out correct version of a three part version number is not passed -find_version_from_git_tags() { - check_packages git; - local variable_name=$1 - local requested_version=${!variable_name} - if [ "${requested_version}" = "none" ]; then return; fi - local repository=$2 - local prefix=${3:-"tags/v"} - local separator=${4:-"."} - local suffix=${5:-} - local last_part_optional=${6:-"false"} - if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then - local escaped_separator=${separator//./\\.} - local last_part= - if [ "${last_part_optional}" = "true" ]; then - last_part+="(${escaped_separator}[0-9]+)?" - last_part+="(${escaped_separator}[0-9]+)?" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})?" - fi - else - last_part+="${escaped_separator}[0-9]+" - last_part+="${escaped_separator}[0-9]+" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})" - fi - fi - local regex="${prefix}\\K[0-9]+${last_part}$" - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then - declare -g "${variable_name}"="$(echo "${version_list}" | head -n 1)" - else - set +e - declare -g "${variable_name}"="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" - set -e - fi - fi - if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then - echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 - exit 1 - fi - echo "${variable_name}=${!variable_name}" -} - -export -f find_version_from_git_tags; - -# Use semver logic to decrement a version number then look for the closest match -find_prev_version_from_git_tags() { - local variable_name=$1 - local current_version=${!variable_name} - local repository=$2 - # Normally a "v" is used before the version number, but support alternate cases - local prefix=${3:-"tags/v"} - # Some repositories use "_" instead of "." for version number part separation, support that - local separator=${4:-"."} - # Some repositories may have tags that include a suffix (e.g. actions/node-versions) - local version_suffix_regex=${5:-} - # Some tools release versions that omit the last digit (e.g. go) - local last_part_optional=${6:-"false"} - # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. - set +e - major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" - minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" - breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" - - if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then - ((major=major-1)) - declare -g "${variable_name}"="${major}" - # Look for latest version from previous major release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - # Handle situations like Go's odd version pattern where "0" releases omit the last part - elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then - ((minor=minor-1)) - declare -g "${variable_name}"="${major}.${minor}" - # Look for latest version from previous minor release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - else - ((breakfix=breakfix-1)) - if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then - declare -g "${variable_name}"="${major}.${minor}" - else - declare -g "${variable_name}"="${major}.${minor}.${breakfix}" - fi - fi - set -e -} - -export -f find_prev_version_from_git_tags; - # Determine the appropriate non-root user find_non_root_user() { USERNAME="${USERNAME:-"${_REMOTE_USER:-"auto"}"}"; @@ -241,3 +168,19 @@ find_non_root_user() { } export -f find_non_root_user; + +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/find-version-from-git-tags.sh"; + +find_version_from_git_tags() { + check_packages git; + _find_version_from_git_tags "$@"; +} + +export -f find_version_from_git_tags; + +find_prev_version_from_git_tags() { + _find_prev_version_from_git_tags "$@"; +} + +export -f find_prev_version_from_git_tags; diff --git a/features/src/ninja/common/find-version-from-git-tags.sh b/features/src/ninja/common/find-version-from-git-tags.sh new file mode 100644 index 000000000..9e9d649ce --- /dev/null +++ b/features/src/ninja/common/find-version-from-git-tags.sh @@ -0,0 +1,109 @@ +#! /usr/bin/env bash + +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + +# Figure out correct version of a three part version number is not passed +_find_version_from_git_tags() { + local variable_name="$1" + local requested_version="${!variable_name}" + if [ "${requested_version}" = "none" ]; then return; fi + local repository="$2" + local prefix="${3:-"tags/v"}" + local separator="${4:-"."}" + local suffix="${5:-}" + local last_part_optional="${6:-"false"}" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator="${separator//./\\.}" + local last_part="" + if [ "${last_part_optional}" = "true" ]; then + last_part+="(${escaped_separator}[0-9]+)?" + last_part+="(${escaped_separator}[0-9]+)?" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})?" + fi + else + last_part+="${escaped_separator}[0-9]+" + last_part+="${escaped_separator}[0-9]+" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})" + fi + fi + local regex="${prefix}\\K[0-9]+${last_part}$" + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch + fi + local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + fi + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + requested_version="$(head -n 1 <<< "${version_list}")" + else + set +e + requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" + set -e + fi + fi + if [ ! -n "${requested_version:+x}" ] || ! grep "^${requested_version//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + return 1 + fi + _upvar "${variable_name}" "${requested_version}" + echo "${variable_name}=${requested_version}" +} + +# Use semver logic to decrement a version number then look for the closest match +_find_prev_version_from_git_tags() { + local variable_name="$1" + local current_version="${!variable_name}" + local repository="$2" + # Normally a "v" is used before the version number, but support alternate cases + local prefix="${3:-"tags/v"}" + # Some repositories use "_" instead of "." for version number part separation, support that + local separator="${4:-"."}" + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex="${5:-}" + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional="${6:-"false"}" + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + local major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + local minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + local breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + _upvar "${variable_name}" "${major}" + # Look for latest version from previous major release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + _upvar "${variable_name}" "${major}.${minor}" + # Look for latest version from previous minor release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + _upvar "${variable_name}" "${major}.${minor}" + else + _upvar "${variable_name}" "${major}.${minor}.${breakfix}" + fi + fi + set -e +} diff --git a/features/src/ninja/common/utilities.sh b/features/src/ninja/common/utilities.sh index 89879e7bf..49a044b95 100644 --- a/features/src/ninja/common/utilities.sh +++ b/features/src/ninja/common/utilities.sh @@ -2,6 +2,22 @@ # Define common shell functions +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + # Run apt update if apt lists aren't populated apt_get_update() { if [ "$(find /var/lib/apt/lists -mindepth 1 | head -n1 | wc -l)" = "0" ]; then @@ -131,95 +147,6 @@ add_etc_profile_d_script() { export -f add_etc_profile_d_script; -# Figure out correct version of a three part version number is not passed -find_version_from_git_tags() { - check_packages git; - local variable_name=$1 - local requested_version=${!variable_name} - if [ "${requested_version}" = "none" ]; then return; fi - local repository=$2 - local prefix=${3:-"tags/v"} - local separator=${4:-"."} - local suffix=${5:-} - local last_part_optional=${6:-"false"} - if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then - local escaped_separator=${separator//./\\.} - local last_part= - if [ "${last_part_optional}" = "true" ]; then - last_part+="(${escaped_separator}[0-9]+)?" - last_part+="(${escaped_separator}[0-9]+)?" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})?" - fi - else - last_part+="${escaped_separator}[0-9]+" - last_part+="${escaped_separator}[0-9]+" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})" - fi - fi - local regex="${prefix}\\K[0-9]+${last_part}$" - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then - declare -g "${variable_name}"="$(echo "${version_list}" | head -n 1)" - else - set +e - declare -g "${variable_name}"="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" - set -e - fi - fi - if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then - echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 - exit 1 - fi - echo "${variable_name}=${!variable_name}" -} - -export -f find_version_from_git_tags; - -# Use semver logic to decrement a version number then look for the closest match -find_prev_version_from_git_tags() { - local variable_name=$1 - local current_version=${!variable_name} - local repository=$2 - # Normally a "v" is used before the version number, but support alternate cases - local prefix=${3:-"tags/v"} - # Some repositories use "_" instead of "." for version number part separation, support that - local separator=${4:-"."} - # Some repositories may have tags that include a suffix (e.g. actions/node-versions) - local version_suffix_regex=${5:-} - # Some tools release versions that omit the last digit (e.g. go) - local last_part_optional=${6:-"false"} - # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. - set +e - major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" - minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" - breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" - - if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then - ((major=major-1)) - declare -g "${variable_name}"="${major}" - # Look for latest version from previous major release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - # Handle situations like Go's odd version pattern where "0" releases omit the last part - elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then - ((minor=minor-1)) - declare -g "${variable_name}"="${major}.${minor}" - # Look for latest version from previous minor release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - else - ((breakfix=breakfix-1)) - if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then - declare -g "${variable_name}"="${major}.${minor}" - else - declare -g "${variable_name}"="${major}.${minor}.${breakfix}" - fi - fi - set -e -} - -export -f find_prev_version_from_git_tags; - # Determine the appropriate non-root user find_non_root_user() { USERNAME="${USERNAME:-"${_REMOTE_USER:-"auto"}"}"; @@ -241,3 +168,19 @@ find_non_root_user() { } export -f find_non_root_user; + +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/find-version-from-git-tags.sh"; + +find_version_from_git_tags() { + check_packages git; + _find_version_from_git_tags "$@"; +} + +export -f find_version_from_git_tags; + +find_prev_version_from_git_tags() { + _find_prev_version_from_git_tags "$@"; +} + +export -f find_prev_version_from_git_tags; diff --git a/features/src/nvhpc/common/find-version-from-git-tags.sh b/features/src/nvhpc/common/find-version-from-git-tags.sh new file mode 100644 index 000000000..9e9d649ce --- /dev/null +++ b/features/src/nvhpc/common/find-version-from-git-tags.sh @@ -0,0 +1,109 @@ +#! /usr/bin/env bash + +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + +# Figure out correct version of a three part version number is not passed +_find_version_from_git_tags() { + local variable_name="$1" + local requested_version="${!variable_name}" + if [ "${requested_version}" = "none" ]; then return; fi + local repository="$2" + local prefix="${3:-"tags/v"}" + local separator="${4:-"."}" + local suffix="${5:-}" + local last_part_optional="${6:-"false"}" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator="${separator//./\\.}" + local last_part="" + if [ "${last_part_optional}" = "true" ]; then + last_part+="(${escaped_separator}[0-9]+)?" + last_part+="(${escaped_separator}[0-9]+)?" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})?" + fi + else + last_part+="${escaped_separator}[0-9]+" + last_part+="${escaped_separator}[0-9]+" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})" + fi + fi + local regex="${prefix}\\K[0-9]+${last_part}$" + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch + fi + local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + fi + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + requested_version="$(head -n 1 <<< "${version_list}")" + else + set +e + requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" + set -e + fi + fi + if [ ! -n "${requested_version:+x}" ] || ! grep "^${requested_version//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + return 1 + fi + _upvar "${variable_name}" "${requested_version}" + echo "${variable_name}=${requested_version}" +} + +# Use semver logic to decrement a version number then look for the closest match +_find_prev_version_from_git_tags() { + local variable_name="$1" + local current_version="${!variable_name}" + local repository="$2" + # Normally a "v" is used before the version number, but support alternate cases + local prefix="${3:-"tags/v"}" + # Some repositories use "_" instead of "." for version number part separation, support that + local separator="${4:-"."}" + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex="${5:-}" + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional="${6:-"false"}" + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + local major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + local minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + local breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + _upvar "${variable_name}" "${major}" + # Look for latest version from previous major release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + _upvar "${variable_name}" "${major}.${minor}" + # Look for latest version from previous minor release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + _upvar "${variable_name}" "${major}.${minor}" + else + _upvar "${variable_name}" "${major}.${minor}.${breakfix}" + fi + fi + set -e +} diff --git a/features/src/nvhpc/common/utilities.sh b/features/src/nvhpc/common/utilities.sh index 89879e7bf..49a044b95 100644 --- a/features/src/nvhpc/common/utilities.sh +++ b/features/src/nvhpc/common/utilities.sh @@ -2,6 +2,22 @@ # Define common shell functions +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + # Run apt update if apt lists aren't populated apt_get_update() { if [ "$(find /var/lib/apt/lists -mindepth 1 | head -n1 | wc -l)" = "0" ]; then @@ -131,95 +147,6 @@ add_etc_profile_d_script() { export -f add_etc_profile_d_script; -# Figure out correct version of a three part version number is not passed -find_version_from_git_tags() { - check_packages git; - local variable_name=$1 - local requested_version=${!variable_name} - if [ "${requested_version}" = "none" ]; then return; fi - local repository=$2 - local prefix=${3:-"tags/v"} - local separator=${4:-"."} - local suffix=${5:-} - local last_part_optional=${6:-"false"} - if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then - local escaped_separator=${separator//./\\.} - local last_part= - if [ "${last_part_optional}" = "true" ]; then - last_part+="(${escaped_separator}[0-9]+)?" - last_part+="(${escaped_separator}[0-9]+)?" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})?" - fi - else - last_part+="${escaped_separator}[0-9]+" - last_part+="${escaped_separator}[0-9]+" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})" - fi - fi - local regex="${prefix}\\K[0-9]+${last_part}$" - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then - declare -g "${variable_name}"="$(echo "${version_list}" | head -n 1)" - else - set +e - declare -g "${variable_name}"="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" - set -e - fi - fi - if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then - echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 - exit 1 - fi - echo "${variable_name}=${!variable_name}" -} - -export -f find_version_from_git_tags; - -# Use semver logic to decrement a version number then look for the closest match -find_prev_version_from_git_tags() { - local variable_name=$1 - local current_version=${!variable_name} - local repository=$2 - # Normally a "v" is used before the version number, but support alternate cases - local prefix=${3:-"tags/v"} - # Some repositories use "_" instead of "." for version number part separation, support that - local separator=${4:-"."} - # Some repositories may have tags that include a suffix (e.g. actions/node-versions) - local version_suffix_regex=${5:-} - # Some tools release versions that omit the last digit (e.g. go) - local last_part_optional=${6:-"false"} - # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. - set +e - major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" - minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" - breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" - - if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then - ((major=major-1)) - declare -g "${variable_name}"="${major}" - # Look for latest version from previous major release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - # Handle situations like Go's odd version pattern where "0" releases omit the last part - elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then - ((minor=minor-1)) - declare -g "${variable_name}"="${major}.${minor}" - # Look for latest version from previous minor release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - else - ((breakfix=breakfix-1)) - if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then - declare -g "${variable_name}"="${major}.${minor}" - else - declare -g "${variable_name}"="${major}.${minor}.${breakfix}" - fi - fi - set -e -} - -export -f find_prev_version_from_git_tags; - # Determine the appropriate non-root user find_non_root_user() { USERNAME="${USERNAME:-"${_REMOTE_USER:-"auto"}"}"; @@ -241,3 +168,19 @@ find_non_root_user() { } export -f find_non_root_user; + +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/find-version-from-git-tags.sh"; + +find_version_from_git_tags() { + check_packages git; + _find_version_from_git_tags "$@"; +} + +export -f find_version_from_git_tags; + +find_prev_version_from_git_tags() { + _find_prev_version_from_git_tags "$@"; +} + +export -f find_prev_version_from_git_tags; diff --git a/features/src/oneapi/common/find-version-from-git-tags.sh b/features/src/oneapi/common/find-version-from-git-tags.sh new file mode 100644 index 000000000..9e9d649ce --- /dev/null +++ b/features/src/oneapi/common/find-version-from-git-tags.sh @@ -0,0 +1,109 @@ +#! /usr/bin/env bash + +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + +# Figure out correct version of a three part version number is not passed +_find_version_from_git_tags() { + local variable_name="$1" + local requested_version="${!variable_name}" + if [ "${requested_version}" = "none" ]; then return; fi + local repository="$2" + local prefix="${3:-"tags/v"}" + local separator="${4:-"."}" + local suffix="${5:-}" + local last_part_optional="${6:-"false"}" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator="${separator//./\\.}" + local last_part="" + if [ "${last_part_optional}" = "true" ]; then + last_part+="(${escaped_separator}[0-9]+)?" + last_part+="(${escaped_separator}[0-9]+)?" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})?" + fi + else + last_part+="${escaped_separator}[0-9]+" + last_part+="${escaped_separator}[0-9]+" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})" + fi + fi + local regex="${prefix}\\K[0-9]+${last_part}$" + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch + fi + local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + fi + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + requested_version="$(head -n 1 <<< "${version_list}")" + else + set +e + requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" + set -e + fi + fi + if [ ! -n "${requested_version:+x}" ] || ! grep "^${requested_version//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + return 1 + fi + _upvar "${variable_name}" "${requested_version}" + echo "${variable_name}=${requested_version}" +} + +# Use semver logic to decrement a version number then look for the closest match +_find_prev_version_from_git_tags() { + local variable_name="$1" + local current_version="${!variable_name}" + local repository="$2" + # Normally a "v" is used before the version number, but support alternate cases + local prefix="${3:-"tags/v"}" + # Some repositories use "_" instead of "." for version number part separation, support that + local separator="${4:-"."}" + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex="${5:-}" + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional="${6:-"false"}" + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + local major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + local minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + local breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + _upvar "${variable_name}" "${major}" + # Look for latest version from previous major release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + _upvar "${variable_name}" "${major}.${minor}" + # Look for latest version from previous minor release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + _upvar "${variable_name}" "${major}.${minor}" + else + _upvar "${variable_name}" "${major}.${minor}.${breakfix}" + fi + fi + set -e +} diff --git a/features/src/oneapi/common/utilities.sh b/features/src/oneapi/common/utilities.sh index 89879e7bf..49a044b95 100644 --- a/features/src/oneapi/common/utilities.sh +++ b/features/src/oneapi/common/utilities.sh @@ -2,6 +2,22 @@ # Define common shell functions +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + # Run apt update if apt lists aren't populated apt_get_update() { if [ "$(find /var/lib/apt/lists -mindepth 1 | head -n1 | wc -l)" = "0" ]; then @@ -131,95 +147,6 @@ add_etc_profile_d_script() { export -f add_etc_profile_d_script; -# Figure out correct version of a three part version number is not passed -find_version_from_git_tags() { - check_packages git; - local variable_name=$1 - local requested_version=${!variable_name} - if [ "${requested_version}" = "none" ]; then return; fi - local repository=$2 - local prefix=${3:-"tags/v"} - local separator=${4:-"."} - local suffix=${5:-} - local last_part_optional=${6:-"false"} - if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then - local escaped_separator=${separator//./\\.} - local last_part= - if [ "${last_part_optional}" = "true" ]; then - last_part+="(${escaped_separator}[0-9]+)?" - last_part+="(${escaped_separator}[0-9]+)?" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})?" - fi - else - last_part+="${escaped_separator}[0-9]+" - last_part+="${escaped_separator}[0-9]+" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})" - fi - fi - local regex="${prefix}\\K[0-9]+${last_part}$" - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then - declare -g "${variable_name}"="$(echo "${version_list}" | head -n 1)" - else - set +e - declare -g "${variable_name}"="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" - set -e - fi - fi - if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then - echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 - exit 1 - fi - echo "${variable_name}=${!variable_name}" -} - -export -f find_version_from_git_tags; - -# Use semver logic to decrement a version number then look for the closest match -find_prev_version_from_git_tags() { - local variable_name=$1 - local current_version=${!variable_name} - local repository=$2 - # Normally a "v" is used before the version number, but support alternate cases - local prefix=${3:-"tags/v"} - # Some repositories use "_" instead of "." for version number part separation, support that - local separator=${4:-"."} - # Some repositories may have tags that include a suffix (e.g. actions/node-versions) - local version_suffix_regex=${5:-} - # Some tools release versions that omit the last digit (e.g. go) - local last_part_optional=${6:-"false"} - # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. - set +e - major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" - minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" - breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" - - if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then - ((major=major-1)) - declare -g "${variable_name}"="${major}" - # Look for latest version from previous major release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - # Handle situations like Go's odd version pattern where "0" releases omit the last part - elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then - ((minor=minor-1)) - declare -g "${variable_name}"="${major}.${minor}" - # Look for latest version from previous minor release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - else - ((breakfix=breakfix-1)) - if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then - declare -g "${variable_name}"="${major}.${minor}" - else - declare -g "${variable_name}"="${major}.${minor}.${breakfix}" - fi - fi - set -e -} - -export -f find_prev_version_from_git_tags; - # Determine the appropriate non-root user find_non_root_user() { USERNAME="${USERNAME:-"${_REMOTE_USER:-"auto"}"}"; @@ -241,3 +168,19 @@ find_non_root_user() { } export -f find_non_root_user; + +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/find-version-from-git-tags.sh"; + +find_version_from_git_tags() { + check_packages git; + _find_version_from_git_tags "$@"; +} + +export -f find_version_from_git_tags; + +find_prev_version_from_git_tags() { + _find_prev_version_from_git_tags "$@"; +} + +export -f find_prev_version_from_git_tags; diff --git a/features/src/openmpi/common/find-version-from-git-tags.sh b/features/src/openmpi/common/find-version-from-git-tags.sh new file mode 100644 index 000000000..9e9d649ce --- /dev/null +++ b/features/src/openmpi/common/find-version-from-git-tags.sh @@ -0,0 +1,109 @@ +#! /usr/bin/env bash + +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + +# Figure out correct version of a three part version number is not passed +_find_version_from_git_tags() { + local variable_name="$1" + local requested_version="${!variable_name}" + if [ "${requested_version}" = "none" ]; then return; fi + local repository="$2" + local prefix="${3:-"tags/v"}" + local separator="${4:-"."}" + local suffix="${5:-}" + local last_part_optional="${6:-"false"}" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator="${separator//./\\.}" + local last_part="" + if [ "${last_part_optional}" = "true" ]; then + last_part+="(${escaped_separator}[0-9]+)?" + last_part+="(${escaped_separator}[0-9]+)?" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})?" + fi + else + last_part+="${escaped_separator}[0-9]+" + last_part+="${escaped_separator}[0-9]+" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})" + fi + fi + local regex="${prefix}\\K[0-9]+${last_part}$" + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch + fi + local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + fi + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + requested_version="$(head -n 1 <<< "${version_list}")" + else + set +e + requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" + set -e + fi + fi + if [ ! -n "${requested_version:+x}" ] || ! grep "^${requested_version//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + return 1 + fi + _upvar "${variable_name}" "${requested_version}" + echo "${variable_name}=${requested_version}" +} + +# Use semver logic to decrement a version number then look for the closest match +_find_prev_version_from_git_tags() { + local variable_name="$1" + local current_version="${!variable_name}" + local repository="$2" + # Normally a "v" is used before the version number, but support alternate cases + local prefix="${3:-"tags/v"}" + # Some repositories use "_" instead of "." for version number part separation, support that + local separator="${4:-"."}" + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex="${5:-}" + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional="${6:-"false"}" + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + local major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + local minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + local breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + _upvar "${variable_name}" "${major}" + # Look for latest version from previous major release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + _upvar "${variable_name}" "${major}.${minor}" + # Look for latest version from previous minor release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + _upvar "${variable_name}" "${major}.${minor}" + else + _upvar "${variable_name}" "${major}.${minor}.${breakfix}" + fi + fi + set -e +} diff --git a/features/src/openmpi/common/utilities.sh b/features/src/openmpi/common/utilities.sh index 89879e7bf..49a044b95 100644 --- a/features/src/openmpi/common/utilities.sh +++ b/features/src/openmpi/common/utilities.sh @@ -2,6 +2,22 @@ # Define common shell functions +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + # Run apt update if apt lists aren't populated apt_get_update() { if [ "$(find /var/lib/apt/lists -mindepth 1 | head -n1 | wc -l)" = "0" ]; then @@ -131,95 +147,6 @@ add_etc_profile_d_script() { export -f add_etc_profile_d_script; -# Figure out correct version of a three part version number is not passed -find_version_from_git_tags() { - check_packages git; - local variable_name=$1 - local requested_version=${!variable_name} - if [ "${requested_version}" = "none" ]; then return; fi - local repository=$2 - local prefix=${3:-"tags/v"} - local separator=${4:-"."} - local suffix=${5:-} - local last_part_optional=${6:-"false"} - if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then - local escaped_separator=${separator//./\\.} - local last_part= - if [ "${last_part_optional}" = "true" ]; then - last_part+="(${escaped_separator}[0-9]+)?" - last_part+="(${escaped_separator}[0-9]+)?" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})?" - fi - else - last_part+="${escaped_separator}[0-9]+" - last_part+="${escaped_separator}[0-9]+" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})" - fi - fi - local regex="${prefix}\\K[0-9]+${last_part}$" - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then - declare -g "${variable_name}"="$(echo "${version_list}" | head -n 1)" - else - set +e - declare -g "${variable_name}"="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" - set -e - fi - fi - if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then - echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 - exit 1 - fi - echo "${variable_name}=${!variable_name}" -} - -export -f find_version_from_git_tags; - -# Use semver logic to decrement a version number then look for the closest match -find_prev_version_from_git_tags() { - local variable_name=$1 - local current_version=${!variable_name} - local repository=$2 - # Normally a "v" is used before the version number, but support alternate cases - local prefix=${3:-"tags/v"} - # Some repositories use "_" instead of "." for version number part separation, support that - local separator=${4:-"."} - # Some repositories may have tags that include a suffix (e.g. actions/node-versions) - local version_suffix_regex=${5:-} - # Some tools release versions that omit the last digit (e.g. go) - local last_part_optional=${6:-"false"} - # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. - set +e - major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" - minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" - breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" - - if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then - ((major=major-1)) - declare -g "${variable_name}"="${major}" - # Look for latest version from previous major release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - # Handle situations like Go's odd version pattern where "0" releases omit the last part - elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then - ((minor=minor-1)) - declare -g "${variable_name}"="${major}.${minor}" - # Look for latest version from previous minor release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - else - ((breakfix=breakfix-1)) - if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then - declare -g "${variable_name}"="${major}.${minor}" - else - declare -g "${variable_name}"="${major}.${minor}.${breakfix}" - fi - fi - set -e -} - -export -f find_prev_version_from_git_tags; - # Determine the appropriate non-root user find_non_root_user() { USERNAME="${USERNAME:-"${_REMOTE_USER:-"auto"}"}"; @@ -241,3 +168,19 @@ find_non_root_user() { } export -f find_non_root_user; + +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/find-version-from-git-tags.sh"; + +find_version_from_git_tags() { + check_packages git; + _find_version_from_git_tags "$@"; +} + +export -f find_version_from_git_tags; + +find_prev_version_from_git_tags() { + _find_prev_version_from_git_tags "$@"; +} + +export -f find_prev_version_from_git_tags; diff --git a/features/src/rapids-build-utils/common/find-version-from-git-tags.sh b/features/src/rapids-build-utils/common/find-version-from-git-tags.sh new file mode 100644 index 000000000..9e9d649ce --- /dev/null +++ b/features/src/rapids-build-utils/common/find-version-from-git-tags.sh @@ -0,0 +1,109 @@ +#! /usr/bin/env bash + +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + +# Figure out correct version of a three part version number is not passed +_find_version_from_git_tags() { + local variable_name="$1" + local requested_version="${!variable_name}" + if [ "${requested_version}" = "none" ]; then return; fi + local repository="$2" + local prefix="${3:-"tags/v"}" + local separator="${4:-"."}" + local suffix="${5:-}" + local last_part_optional="${6:-"false"}" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator="${separator//./\\.}" + local last_part="" + if [ "${last_part_optional}" = "true" ]; then + last_part+="(${escaped_separator}[0-9]+)?" + last_part+="(${escaped_separator}[0-9]+)?" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})?" + fi + else + last_part+="${escaped_separator}[0-9]+" + last_part+="${escaped_separator}[0-9]+" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})" + fi + fi + local regex="${prefix}\\K[0-9]+${last_part}$" + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch + fi + local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + fi + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + requested_version="$(head -n 1 <<< "${version_list}")" + else + set +e + requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" + set -e + fi + fi + if [ ! -n "${requested_version:+x}" ] || ! grep "^${requested_version//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + return 1 + fi + _upvar "${variable_name}" "${requested_version}" + echo "${variable_name}=${requested_version}" +} + +# Use semver logic to decrement a version number then look for the closest match +_find_prev_version_from_git_tags() { + local variable_name="$1" + local current_version="${!variable_name}" + local repository="$2" + # Normally a "v" is used before the version number, but support alternate cases + local prefix="${3:-"tags/v"}" + # Some repositories use "_" instead of "." for version number part separation, support that + local separator="${4:-"."}" + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex="${5:-}" + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional="${6:-"false"}" + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + local major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + local minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + local breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + _upvar "${variable_name}" "${major}" + # Look for latest version from previous major release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + _upvar "${variable_name}" "${major}.${minor}" + # Look for latest version from previous minor release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + _upvar "${variable_name}" "${major}.${minor}" + else + _upvar "${variable_name}" "${major}.${minor}.${breakfix}" + fi + fi + set -e +} diff --git a/features/src/rapids-build-utils/common/utilities.sh b/features/src/rapids-build-utils/common/utilities.sh index 89879e7bf..49a044b95 100644 --- a/features/src/rapids-build-utils/common/utilities.sh +++ b/features/src/rapids-build-utils/common/utilities.sh @@ -2,6 +2,22 @@ # Define common shell functions +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + # Run apt update if apt lists aren't populated apt_get_update() { if [ "$(find /var/lib/apt/lists -mindepth 1 | head -n1 | wc -l)" = "0" ]; then @@ -131,95 +147,6 @@ add_etc_profile_d_script() { export -f add_etc_profile_d_script; -# Figure out correct version of a three part version number is not passed -find_version_from_git_tags() { - check_packages git; - local variable_name=$1 - local requested_version=${!variable_name} - if [ "${requested_version}" = "none" ]; then return; fi - local repository=$2 - local prefix=${3:-"tags/v"} - local separator=${4:-"."} - local suffix=${5:-} - local last_part_optional=${6:-"false"} - if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then - local escaped_separator=${separator//./\\.} - local last_part= - if [ "${last_part_optional}" = "true" ]; then - last_part+="(${escaped_separator}[0-9]+)?" - last_part+="(${escaped_separator}[0-9]+)?" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})?" - fi - else - last_part+="${escaped_separator}[0-9]+" - last_part+="${escaped_separator}[0-9]+" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})" - fi - fi - local regex="${prefix}\\K[0-9]+${last_part}$" - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then - declare -g "${variable_name}"="$(echo "${version_list}" | head -n 1)" - else - set +e - declare -g "${variable_name}"="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" - set -e - fi - fi - if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then - echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 - exit 1 - fi - echo "${variable_name}=${!variable_name}" -} - -export -f find_version_from_git_tags; - -# Use semver logic to decrement a version number then look for the closest match -find_prev_version_from_git_tags() { - local variable_name=$1 - local current_version=${!variable_name} - local repository=$2 - # Normally a "v" is used before the version number, but support alternate cases - local prefix=${3:-"tags/v"} - # Some repositories use "_" instead of "." for version number part separation, support that - local separator=${4:-"."} - # Some repositories may have tags that include a suffix (e.g. actions/node-versions) - local version_suffix_regex=${5:-} - # Some tools release versions that omit the last digit (e.g. go) - local last_part_optional=${6:-"false"} - # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. - set +e - major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" - minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" - breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" - - if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then - ((major=major-1)) - declare -g "${variable_name}"="${major}" - # Look for latest version from previous major release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - # Handle situations like Go's odd version pattern where "0" releases omit the last part - elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then - ((minor=minor-1)) - declare -g "${variable_name}"="${major}.${minor}" - # Look for latest version from previous minor release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - else - ((breakfix=breakfix-1)) - if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then - declare -g "${variable_name}"="${major}.${minor}" - else - declare -g "${variable_name}"="${major}.${minor}.${breakfix}" - fi - fi - set -e -} - -export -f find_prev_version_from_git_tags; - # Determine the appropriate non-root user find_non_root_user() { USERNAME="${USERNAME:-"${_REMOTE_USER:-"auto"}"}"; @@ -241,3 +168,19 @@ find_non_root_user() { } export -f find_non_root_user; + +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/find-version-from-git-tags.sh"; + +find_version_from_git_tags() { + check_packages git; + _find_version_from_git_tags "$@"; +} + +export -f find_version_from_git_tags; + +find_prev_version_from_git_tags() { + _find_prev_version_from_git_tags "$@"; +} + +export -f find_prev_version_from_git_tags; diff --git a/features/src/rust/common/find-version-from-git-tags.sh b/features/src/rust/common/find-version-from-git-tags.sh new file mode 100644 index 000000000..9e9d649ce --- /dev/null +++ b/features/src/rust/common/find-version-from-git-tags.sh @@ -0,0 +1,109 @@ +#! /usr/bin/env bash + +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + +# Figure out correct version of a three part version number is not passed +_find_version_from_git_tags() { + local variable_name="$1" + local requested_version="${!variable_name}" + if [ "${requested_version}" = "none" ]; then return; fi + local repository="$2" + local prefix="${3:-"tags/v"}" + local separator="${4:-"."}" + local suffix="${5:-}" + local last_part_optional="${6:-"false"}" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator="${separator//./\\.}" + local last_part="" + if [ "${last_part_optional}" = "true" ]; then + last_part+="(${escaped_separator}[0-9]+)?" + last_part+="(${escaped_separator}[0-9]+)?" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})?" + fi + else + last_part+="${escaped_separator}[0-9]+" + last_part+="${escaped_separator}[0-9]+" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})" + fi + fi + local regex="${prefix}\\K[0-9]+${last_part}$" + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch + fi + local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + fi + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + requested_version="$(head -n 1 <<< "${version_list}")" + else + set +e + requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" + set -e + fi + fi + if [ ! -n "${requested_version:+x}" ] || ! grep "^${requested_version//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + return 1 + fi + _upvar "${variable_name}" "${requested_version}" + echo "${variable_name}=${requested_version}" +} + +# Use semver logic to decrement a version number then look for the closest match +_find_prev_version_from_git_tags() { + local variable_name="$1" + local current_version="${!variable_name}" + local repository="$2" + # Normally a "v" is used before the version number, but support alternate cases + local prefix="${3:-"tags/v"}" + # Some repositories use "_" instead of "." for version number part separation, support that + local separator="${4:-"."}" + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex="${5:-}" + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional="${6:-"false"}" + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + local major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + local minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + local breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + _upvar "${variable_name}" "${major}" + # Look for latest version from previous major release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + _upvar "${variable_name}" "${major}.${minor}" + # Look for latest version from previous minor release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + _upvar "${variable_name}" "${major}.${minor}" + else + _upvar "${variable_name}" "${major}.${minor}.${breakfix}" + fi + fi + set -e +} diff --git a/features/src/rust/common/utilities.sh b/features/src/rust/common/utilities.sh index 89879e7bf..49a044b95 100644 --- a/features/src/rust/common/utilities.sh +++ b/features/src/rust/common/utilities.sh @@ -2,6 +2,22 @@ # Define common shell functions +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + # Run apt update if apt lists aren't populated apt_get_update() { if [ "$(find /var/lib/apt/lists -mindepth 1 | head -n1 | wc -l)" = "0" ]; then @@ -131,95 +147,6 @@ add_etc_profile_d_script() { export -f add_etc_profile_d_script; -# Figure out correct version of a three part version number is not passed -find_version_from_git_tags() { - check_packages git; - local variable_name=$1 - local requested_version=${!variable_name} - if [ "${requested_version}" = "none" ]; then return; fi - local repository=$2 - local prefix=${3:-"tags/v"} - local separator=${4:-"."} - local suffix=${5:-} - local last_part_optional=${6:-"false"} - if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then - local escaped_separator=${separator//./\\.} - local last_part= - if [ "${last_part_optional}" = "true" ]; then - last_part+="(${escaped_separator}[0-9]+)?" - last_part+="(${escaped_separator}[0-9]+)?" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})?" - fi - else - last_part+="${escaped_separator}[0-9]+" - last_part+="${escaped_separator}[0-9]+" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})" - fi - fi - local regex="${prefix}\\K[0-9]+${last_part}$" - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then - declare -g "${variable_name}"="$(echo "${version_list}" | head -n 1)" - else - set +e - declare -g "${variable_name}"="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" - set -e - fi - fi - if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then - echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 - exit 1 - fi - echo "${variable_name}=${!variable_name}" -} - -export -f find_version_from_git_tags; - -# Use semver logic to decrement a version number then look for the closest match -find_prev_version_from_git_tags() { - local variable_name=$1 - local current_version=${!variable_name} - local repository=$2 - # Normally a "v" is used before the version number, but support alternate cases - local prefix=${3:-"tags/v"} - # Some repositories use "_" instead of "." for version number part separation, support that - local separator=${4:-"."} - # Some repositories may have tags that include a suffix (e.g. actions/node-versions) - local version_suffix_regex=${5:-} - # Some tools release versions that omit the last digit (e.g. go) - local last_part_optional=${6:-"false"} - # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. - set +e - major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" - minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" - breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" - - if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then - ((major=major-1)) - declare -g "${variable_name}"="${major}" - # Look for latest version from previous major release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - # Handle situations like Go's odd version pattern where "0" releases omit the last part - elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then - ((minor=minor-1)) - declare -g "${variable_name}"="${major}.${minor}" - # Look for latest version from previous minor release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - else - ((breakfix=breakfix-1)) - if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then - declare -g "${variable_name}"="${major}.${minor}" - else - declare -g "${variable_name}"="${major}.${minor}.${breakfix}" - fi - fi - set -e -} - -export -f find_prev_version_from_git_tags; - # Determine the appropriate non-root user find_non_root_user() { USERNAME="${USERNAME:-"${_REMOTE_USER:-"auto"}"}"; @@ -241,3 +168,19 @@ find_non_root_user() { } export -f find_non_root_user; + +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/find-version-from-git-tags.sh"; + +find_version_from_git_tags() { + check_packages git; + _find_version_from_git_tags "$@"; +} + +export -f find_version_from_git_tags; + +find_prev_version_from_git_tags() { + _find_prev_version_from_git_tags "$@"; +} + +export -f find_prev_version_from_git_tags; diff --git a/features/src/sccache/common/find-version-from-git-tags.sh b/features/src/sccache/common/find-version-from-git-tags.sh new file mode 100644 index 000000000..9e9d649ce --- /dev/null +++ b/features/src/sccache/common/find-version-from-git-tags.sh @@ -0,0 +1,109 @@ +#! /usr/bin/env bash + +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + +# Figure out correct version of a three part version number is not passed +_find_version_from_git_tags() { + local variable_name="$1" + local requested_version="${!variable_name}" + if [ "${requested_version}" = "none" ]; then return; fi + local repository="$2" + local prefix="${3:-"tags/v"}" + local separator="${4:-"."}" + local suffix="${5:-}" + local last_part_optional="${6:-"false"}" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator="${separator//./\\.}" + local last_part="" + if [ "${last_part_optional}" = "true" ]; then + last_part+="(${escaped_separator}[0-9]+)?" + last_part+="(${escaped_separator}[0-9]+)?" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})?" + fi + else + last_part+="${escaped_separator}[0-9]+" + last_part+="${escaped_separator}[0-9]+" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})" + fi + fi + local regex="${prefix}\\K[0-9]+${last_part}$" + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch + fi + local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + fi + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + requested_version="$(head -n 1 <<< "${version_list}")" + else + set +e + requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" + set -e + fi + fi + if [ ! -n "${requested_version:+x}" ] || ! grep "^${requested_version//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + return 1 + fi + _upvar "${variable_name}" "${requested_version}" + echo "${variable_name}=${requested_version}" +} + +# Use semver logic to decrement a version number then look for the closest match +_find_prev_version_from_git_tags() { + local variable_name="$1" + local current_version="${!variable_name}" + local repository="$2" + # Normally a "v" is used before the version number, but support alternate cases + local prefix="${3:-"tags/v"}" + # Some repositories use "_" instead of "." for version number part separation, support that + local separator="${4:-"."}" + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex="${5:-}" + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional="${6:-"false"}" + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + local major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + local minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + local breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + _upvar "${variable_name}" "${major}" + # Look for latest version from previous major release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + _upvar "${variable_name}" "${major}.${minor}" + # Look for latest version from previous minor release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + _upvar "${variable_name}" "${major}.${minor}" + else + _upvar "${variable_name}" "${major}.${minor}.${breakfix}" + fi + fi + set -e +} diff --git a/features/src/sccache/common/utilities.sh b/features/src/sccache/common/utilities.sh index 89879e7bf..49a044b95 100644 --- a/features/src/sccache/common/utilities.sh +++ b/features/src/sccache/common/utilities.sh @@ -2,6 +2,22 @@ # Define common shell functions +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + # Run apt update if apt lists aren't populated apt_get_update() { if [ "$(find /var/lib/apt/lists -mindepth 1 | head -n1 | wc -l)" = "0" ]; then @@ -131,95 +147,6 @@ add_etc_profile_d_script() { export -f add_etc_profile_d_script; -# Figure out correct version of a three part version number is not passed -find_version_from_git_tags() { - check_packages git; - local variable_name=$1 - local requested_version=${!variable_name} - if [ "${requested_version}" = "none" ]; then return; fi - local repository=$2 - local prefix=${3:-"tags/v"} - local separator=${4:-"."} - local suffix=${5:-} - local last_part_optional=${6:-"false"} - if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then - local escaped_separator=${separator//./\\.} - local last_part= - if [ "${last_part_optional}" = "true" ]; then - last_part+="(${escaped_separator}[0-9]+)?" - last_part+="(${escaped_separator}[0-9]+)?" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})?" - fi - else - last_part+="${escaped_separator}[0-9]+" - last_part+="${escaped_separator}[0-9]+" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})" - fi - fi - local regex="${prefix}\\K[0-9]+${last_part}$" - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then - declare -g "${variable_name}"="$(echo "${version_list}" | head -n 1)" - else - set +e - declare -g "${variable_name}"="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" - set -e - fi - fi - if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then - echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 - exit 1 - fi - echo "${variable_name}=${!variable_name}" -} - -export -f find_version_from_git_tags; - -# Use semver logic to decrement a version number then look for the closest match -find_prev_version_from_git_tags() { - local variable_name=$1 - local current_version=${!variable_name} - local repository=$2 - # Normally a "v" is used before the version number, but support alternate cases - local prefix=${3:-"tags/v"} - # Some repositories use "_" instead of "." for version number part separation, support that - local separator=${4:-"."} - # Some repositories may have tags that include a suffix (e.g. actions/node-versions) - local version_suffix_regex=${5:-} - # Some tools release versions that omit the last digit (e.g. go) - local last_part_optional=${6:-"false"} - # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. - set +e - major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" - minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" - breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" - - if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then - ((major=major-1)) - declare -g "${variable_name}"="${major}" - # Look for latest version from previous major release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - # Handle situations like Go's odd version pattern where "0" releases omit the last part - elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then - ((minor=minor-1)) - declare -g "${variable_name}"="${major}.${minor}" - # Look for latest version from previous minor release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - else - ((breakfix=breakfix-1)) - if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then - declare -g "${variable_name}"="${major}.${minor}" - else - declare -g "${variable_name}"="${major}.${minor}.${breakfix}" - fi - fi - set -e -} - -export -f find_prev_version_from_git_tags; - # Determine the appropriate non-root user find_non_root_user() { USERNAME="${USERNAME:-"${_REMOTE_USER:-"auto"}"}"; @@ -241,3 +168,19 @@ find_non_root_user() { } export -f find_non_root_user; + +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/find-version-from-git-tags.sh"; + +find_version_from_git_tags() { + check_packages git; + _find_version_from_git_tags "$@"; +} + +export -f find_version_from_git_tags; + +find_prev_version_from_git_tags() { + _find_prev_version_from_git_tags "$@"; +} + +export -f find_prev_version_from_git_tags; diff --git a/features/src/ucx/common/find-version-from-git-tags.sh b/features/src/ucx/common/find-version-from-git-tags.sh new file mode 100644 index 000000000..9e9d649ce --- /dev/null +++ b/features/src/ucx/common/find-version-from-git-tags.sh @@ -0,0 +1,109 @@ +#! /usr/bin/env bash + +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + +# Figure out correct version of a three part version number is not passed +_find_version_from_git_tags() { + local variable_name="$1" + local requested_version="${!variable_name}" + if [ "${requested_version}" = "none" ]; then return; fi + local repository="$2" + local prefix="${3:-"tags/v"}" + local separator="${4:-"."}" + local suffix="${5:-}" + local last_part_optional="${6:-"false"}" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator="${separator//./\\.}" + local last_part="" + if [ "${last_part_optional}" = "true" ]; then + last_part+="(${escaped_separator}[0-9]+)?" + last_part+="(${escaped_separator}[0-9]+)?" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})?" + fi + else + last_part+="${escaped_separator}[0-9]+" + last_part+="${escaped_separator}[0-9]+" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})" + fi + fi + local regex="${prefix}\\K[0-9]+${last_part}$" + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch + fi + local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + fi + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + requested_version="$(head -n 1 <<< "${version_list}")" + else + set +e + requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" + set -e + fi + fi + if [ ! -n "${requested_version:+x}" ] || ! grep "^${requested_version//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + return 1 + fi + _upvar "${variable_name}" "${requested_version}" + echo "${variable_name}=${requested_version}" +} + +# Use semver logic to decrement a version number then look for the closest match +_find_prev_version_from_git_tags() { + local variable_name="$1" + local current_version="${!variable_name}" + local repository="$2" + # Normally a "v" is used before the version number, but support alternate cases + local prefix="${3:-"tags/v"}" + # Some repositories use "_" instead of "." for version number part separation, support that + local separator="${4:-"."}" + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex="${5:-}" + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional="${6:-"false"}" + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + local major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + local minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + local breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + _upvar "${variable_name}" "${major}" + # Look for latest version from previous major release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + _upvar "${variable_name}" "${major}.${minor}" + # Look for latest version from previous minor release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + _upvar "${variable_name}" "${major}.${minor}" + else + _upvar "${variable_name}" "${major}.${minor}.${breakfix}" + fi + fi + set -e +} diff --git a/features/src/ucx/common/utilities.sh b/features/src/ucx/common/utilities.sh index 89879e7bf..49a044b95 100644 --- a/features/src/ucx/common/utilities.sh +++ b/features/src/ucx/common/utilities.sh @@ -2,6 +2,22 @@ # Define common shell functions +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + # Run apt update if apt lists aren't populated apt_get_update() { if [ "$(find /var/lib/apt/lists -mindepth 1 | head -n1 | wc -l)" = "0" ]; then @@ -131,95 +147,6 @@ add_etc_profile_d_script() { export -f add_etc_profile_d_script; -# Figure out correct version of a three part version number is not passed -find_version_from_git_tags() { - check_packages git; - local variable_name=$1 - local requested_version=${!variable_name} - if [ "${requested_version}" = "none" ]; then return; fi - local repository=$2 - local prefix=${3:-"tags/v"} - local separator=${4:-"."} - local suffix=${5:-} - local last_part_optional=${6:-"false"} - if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then - local escaped_separator=${separator//./\\.} - local last_part= - if [ "${last_part_optional}" = "true" ]; then - last_part+="(${escaped_separator}[0-9]+)?" - last_part+="(${escaped_separator}[0-9]+)?" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})?" - fi - else - last_part+="${escaped_separator}[0-9]+" - last_part+="${escaped_separator}[0-9]+" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})" - fi - fi - local regex="${prefix}\\K[0-9]+${last_part}$" - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then - declare -g "${variable_name}"="$(echo "${version_list}" | head -n 1)" - else - set +e - declare -g "${variable_name}"="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" - set -e - fi - fi - if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then - echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 - exit 1 - fi - echo "${variable_name}=${!variable_name}" -} - -export -f find_version_from_git_tags; - -# Use semver logic to decrement a version number then look for the closest match -find_prev_version_from_git_tags() { - local variable_name=$1 - local current_version=${!variable_name} - local repository=$2 - # Normally a "v" is used before the version number, but support alternate cases - local prefix=${3:-"tags/v"} - # Some repositories use "_" instead of "." for version number part separation, support that - local separator=${4:-"."} - # Some repositories may have tags that include a suffix (e.g. actions/node-versions) - local version_suffix_regex=${5:-} - # Some tools release versions that omit the last digit (e.g. go) - local last_part_optional=${6:-"false"} - # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. - set +e - major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" - minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" - breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" - - if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then - ((major=major-1)) - declare -g "${variable_name}"="${major}" - # Look for latest version from previous major release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - # Handle situations like Go's odd version pattern where "0" releases omit the last part - elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then - ((minor=minor-1)) - declare -g "${variable_name}"="${major}.${minor}" - # Look for latest version from previous minor release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - else - ((breakfix=breakfix-1)) - if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then - declare -g "${variable_name}"="${major}.${minor}" - else - declare -g "${variable_name}"="${major}.${minor}.${breakfix}" - fi - fi - set -e -} - -export -f find_prev_version_from_git_tags; - # Determine the appropriate non-root user find_non_root_user() { USERNAME="${USERNAME:-"${_REMOTE_USER:-"auto"}"}"; @@ -241,3 +168,19 @@ find_non_root_user() { } export -f find_non_root_user; + +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/find-version-from-git-tags.sh"; + +find_version_from_git_tags() { + check_packages git; + _find_version_from_git_tags "$@"; +} + +export -f find_version_from_git_tags; + +find_prev_version_from_git_tags() { + _find_prev_version_from_git_tags "$@"; +} + +export -f find_prev_version_from_git_tags; diff --git a/features/src/utils/common/find-version-from-git-tags.sh b/features/src/utils/common/find-version-from-git-tags.sh new file mode 100644 index 000000000..9e9d649ce --- /dev/null +++ b/features/src/utils/common/find-version-from-git-tags.sh @@ -0,0 +1,109 @@ +#! /usr/bin/env bash + +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + +# Figure out correct version of a three part version number is not passed +_find_version_from_git_tags() { + local variable_name="$1" + local requested_version="${!variable_name}" + if [ "${requested_version}" = "none" ]; then return; fi + local repository="$2" + local prefix="${3:-"tags/v"}" + local separator="${4:-"."}" + local suffix="${5:-}" + local last_part_optional="${6:-"false"}" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator="${separator//./\\.}" + local last_part="" + if [ "${last_part_optional}" = "true" ]; then + last_part+="(${escaped_separator}[0-9]+)?" + last_part+="(${escaped_separator}[0-9]+)?" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})?" + fi + else + last_part+="${escaped_separator}[0-9]+" + last_part+="${escaped_separator}[0-9]+" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})" + fi + fi + local regex="${prefix}\\K[0-9]+${last_part}$" + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch + fi + local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + fi + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + requested_version="$(head -n 1 <<< "${version_list}")" + else + set +e + requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" + set -e + fi + fi + if [ ! -n "${requested_version:+x}" ] || ! grep "^${requested_version//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + return 1 + fi + _upvar "${variable_name}" "${requested_version}" + echo "${variable_name}=${requested_version}" +} + +# Use semver logic to decrement a version number then look for the closest match +_find_prev_version_from_git_tags() { + local variable_name="$1" + local current_version="${!variable_name}" + local repository="$2" + # Normally a "v" is used before the version number, but support alternate cases + local prefix="${3:-"tags/v"}" + # Some repositories use "_" instead of "." for version number part separation, support that + local separator="${4:-"."}" + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex="${5:-}" + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional="${6:-"false"}" + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + local major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + local minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + local breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + _upvar "${variable_name}" "${major}" + # Look for latest version from previous major release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + _upvar "${variable_name}" "${major}.${minor}" + # Look for latest version from previous minor release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + _upvar "${variable_name}" "${major}.${minor}" + else + _upvar "${variable_name}" "${major}.${minor}.${breakfix}" + fi + fi + set -e +} diff --git a/features/src/utils/common/utilities.sh b/features/src/utils/common/utilities.sh index 89879e7bf..49a044b95 100644 --- a/features/src/utils/common/utilities.sh +++ b/features/src/utils/common/utilities.sh @@ -2,6 +2,22 @@ # Define common shell functions +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + # Run apt update if apt lists aren't populated apt_get_update() { if [ "$(find /var/lib/apt/lists -mindepth 1 | head -n1 | wc -l)" = "0" ]; then @@ -131,95 +147,6 @@ add_etc_profile_d_script() { export -f add_etc_profile_d_script; -# Figure out correct version of a three part version number is not passed -find_version_from_git_tags() { - check_packages git; - local variable_name=$1 - local requested_version=${!variable_name} - if [ "${requested_version}" = "none" ]; then return; fi - local repository=$2 - local prefix=${3:-"tags/v"} - local separator=${4:-"."} - local suffix=${5:-} - local last_part_optional=${6:-"false"} - if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then - local escaped_separator=${separator//./\\.} - local last_part= - if [ "${last_part_optional}" = "true" ]; then - last_part+="(${escaped_separator}[0-9]+)?" - last_part+="(${escaped_separator}[0-9]+)?" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})?" - fi - else - last_part+="${escaped_separator}[0-9]+" - last_part+="${escaped_separator}[0-9]+" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})" - fi - fi - local regex="${prefix}\\K[0-9]+${last_part}$" - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then - declare -g "${variable_name}"="$(echo "${version_list}" | head -n 1)" - else - set +e - declare -g "${variable_name}"="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" - set -e - fi - fi - if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then - echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 - exit 1 - fi - echo "${variable_name}=${!variable_name}" -} - -export -f find_version_from_git_tags; - -# Use semver logic to decrement a version number then look for the closest match -find_prev_version_from_git_tags() { - local variable_name=$1 - local current_version=${!variable_name} - local repository=$2 - # Normally a "v" is used before the version number, but support alternate cases - local prefix=${3:-"tags/v"} - # Some repositories use "_" instead of "." for version number part separation, support that - local separator=${4:-"."} - # Some repositories may have tags that include a suffix (e.g. actions/node-versions) - local version_suffix_regex=${5:-} - # Some tools release versions that omit the last digit (e.g. go) - local last_part_optional=${6:-"false"} - # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. - set +e - major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" - minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" - breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" - - if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then - ((major=major-1)) - declare -g "${variable_name}"="${major}" - # Look for latest version from previous major release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - # Handle situations like Go's odd version pattern where "0" releases omit the last part - elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then - ((minor=minor-1)) - declare -g "${variable_name}"="${major}.${minor}" - # Look for latest version from previous minor release - find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" - else - ((breakfix=breakfix-1)) - if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then - declare -g "${variable_name}"="${major}.${minor}" - else - declare -g "${variable_name}"="${major}.${minor}.${breakfix}" - fi - fi - set -e -} - -export -f find_prev_version_from_git_tags; - # Determine the appropriate non-root user find_non_root_user() { USERNAME="${USERNAME:-"${_REMOTE_USER:-"auto"}"}"; @@ -241,3 +168,19 @@ find_non_root_user() { } export -f find_non_root_user; + +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/find-version-from-git-tags.sh"; + +find_version_from_git_tags() { + check_packages git; + _find_version_from_git_tags "$@"; +} + +export -f find_version_from_git_tags; + +find_prev_version_from_git_tags() { + _find_prev_version_from_git_tags "$@"; +} + +export -f find_prev_version_from_git_tags; diff --git a/features/src/utils/opt/devcontainer/bin/post-attach-command.sh b/features/src/utils/opt/devcontainer/bin/post-attach-command.sh index 224eb5621..97003daff 100755 --- a/features/src/utils/opt/devcontainer/bin/post-attach-command.sh +++ b/features/src/utils/opt/devcontainer/bin/post-attach-command.sh @@ -19,7 +19,7 @@ if ! test -n "${SKIP_DEVCONTAINER_UTILS_POST_ATTACH_COMMAND:+x}"; then if test -n "${DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST:+x}"; then devcontainer-utils-install-sccache \ --repo "${SCCACHE_REPOSITORY:-rapidsai/sccache}" \ - --version "${SCCACHE_VERSION:-latest}" \ + --version "${SCCACHE_VERSION:-rapids}" \ ; if test -n "${SCCACHE_DIST_AUTH_TOKEN:+x}"; then devcontainer-utils-init-sccache-dist \ diff --git a/features/src/utils/opt/devcontainer/bin/sccache/find-version-from-git-tags.sh b/features/src/utils/opt/devcontainer/bin/sccache/find-version-from-git-tags.sh new file mode 100644 index 000000000..9e9d649ce --- /dev/null +++ b/features/src/utils/opt/devcontainer/bin/sccache/find-version-from-git-tags.sh @@ -0,0 +1,109 @@ +#! /usr/bin/env bash + +# Assign variable one scope above the caller +# Usage: local "$1" && _upvar $1 "value(s)" +# Param: $1 Variable name to assign value to +# Param: $* Value(s) to assign. If multiple values, an array is +# assigned, otherwise a single value is assigned. +# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference +_upvar() { + if unset -v "$1"; then + if (( $# == 2 )); then + eval $1=\"\$2\"; + else + eval $1=\(\"\${@:2}\"\); + fi; + fi +} + +# Figure out correct version of a three part version number is not passed +_find_version_from_git_tags() { + local variable_name="$1" + local requested_version="${!variable_name}" + if [ "${requested_version}" = "none" ]; then return; fi + local repository="$2" + local prefix="${3:-"tags/v"}" + local separator="${4:-"."}" + local suffix="${5:-}" + local last_part_optional="${6:-"false"}" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator="${separator//./\\.}" + local last_part="" + if [ "${last_part_optional}" = "true" ]; then + last_part+="(${escaped_separator}[0-9]+)?" + last_part+="(${escaped_separator}[0-9]+)?" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})?" + fi + else + last_part+="${escaped_separator}[0-9]+" + last_part+="${escaped_separator}[0-9]+" + if [ -n "${suffix:+x}" ]; then + last_part+="(${suffix})" + fi + fi + local regex="${prefix}\\K[0-9]+${last_part}$" + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch + fi + local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + fi + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + requested_version="$(head -n 1 <<< "${version_list}")" + else + set +e + requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" + set -e + fi + fi + if [ ! -n "${requested_version:+x}" ] || ! grep "^${requested_version//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + return 1 + fi + _upvar "${variable_name}" "${requested_version}" + echo "${variable_name}=${requested_version}" +} + +# Use semver logic to decrement a version number then look for the closest match +_find_prev_version_from_git_tags() { + local variable_name="$1" + local current_version="${!variable_name}" + local repository="$2" + # Normally a "v" is used before the version number, but support alternate cases + local prefix="${3:-"tags/v"}" + # Some repositories use "_" instead of "." for version number part separation, support that + local separator="${4:-"."}" + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex="${5:-}" + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional="${6:-"false"}" + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + local major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + local minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + local breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + _upvar "${variable_name}" "${major}" + # Look for latest version from previous major release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + _upvar "${variable_name}" "${major}.${minor}" + # Look for latest version from previous minor release + _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + _upvar "${variable_name}" "${major}.${minor}" + else + _upvar "${variable_name}" "${major}.${minor}.${breakfix}" + fi + fi + set -e +} diff --git a/features/src/utils/opt/devcontainer/bin/sccache/install.sh b/features/src/utils/opt/devcontainer/bin/sccache/install.sh index 89ba73db1..439c92177 100755 --- a/features/src/utils/opt/devcontainer/bin/sccache/install.sh +++ b/features/src/utils/opt/devcontainer/bin/sccache/install.sh @@ -15,71 +15,8 @@ # (default: ${SCCACHE_VERSION:-latest}) # - -# Assign variable one scope above the caller -# Usage: local "$1" && _upvar $1 "value(s)" -# Param: $1 Variable name to assign value to -# Param: $* Value(s) to assign. If multiple values, an array is -# assigned, otherwise a single value is assigned. -# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference -_upvar() { - if unset -v "$1"; then - if (( $# == 2 )); then - eval $1=\"\$2\"; - else - eval $1=\(\"\${@:2}\"\); - fi; - fi -} - -# Figure out correct version of a three part version number is not passed -_find_version_from_git_tags() { - local variable_name="$1" - local requested_version="${!variable_name}" - if [ "${requested_version}" = "none" ]; then return; fi - local repository="$2" - local prefix="${3:-"tags/v"}" - local separator="${4:-"."}" - local suffix="${5:-}" - local last_part_optional="${6:-"false"}" - if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then - local escaped_separator=${separator//./\\.} - local last_part= - if [ "${last_part_optional}" = "true" ]; then - last_part+="(${escaped_separator}[0-9]+)?" - last_part+="(${escaped_separator}[0-9]+)?" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})?" - fi - else - last_part+="${escaped_separator}[0-9]+" - last_part+="${escaped_separator}[0-9]+" - if [ -n "${suffix}" ]; then - last_part+="(${suffix})" - fi - fi - local regex="${prefix}\\K[0-9]+${last_part}$" - local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; - if test -n "${remote_upstream_fetch:+x}"; then - git config unset --global remote.upstream.fetch - fi - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if test -n "${remote_upstream_fetch:+x}"; then - git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; - fi - if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then - _upvar "${variable_name}" "$(head -n 1 <<< "${version_list}")" - else - set +e - _upvar "${variable_name}" "$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" - set -e - fi - fi - if [ -z "${!variable_name}" ] || ! grep "^${!variable_name//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then - echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 - return 1 - fi -} +# shellcheck disable=SC1091 +. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/find-version-from-git-tags.sh"; _install_sccache() { local -; @@ -90,21 +27,28 @@ _install_sccache() { # shellcheck disable=SC1091 . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'sccache install-sccache'; + while pgrep sccache >/dev/null 2>&1; do + devcontainer-utils-stop-sccache --kill-all; + done + local sccache_version="${version:-"${SCCACHE_VERSION:-"latest"}"}"; local github_repo="${repo:-"${SCCACHE_REPOSITORY:-"mozilla/sccache"}"}"; - _find_version_from_git_tags sccache_version "https://github.com/$github_repo" "" "" "-.*" "true"; + local find_version_args=("https://github.com/$github_repo"); + if test "$sccache_version" = rapids; then + sccache_version=latest; + find_version_args+=("tags/v" "." "-rapids.*" "true"); + fi + + _find_version_from_git_tags sccache_version "${find_version_args[@]}"; - while pgrep sccache >/dev/null 2>&1; do - devcontainer-utils-stop-sccache --kill-all; + while ! wget --no-hsts -q -O- "https://github.com/$github_repo/releases/download/v$sccache_version/sccache-v$sccache_version-$(uname -m)-unknown-linux-musl.tar.gz" \ + | sudo tar -C /usr/bin -zf - --wildcards --strip-components=1 -x '*/sccache' \ + && sudo chmod +x /usr/bin/sccache; do + echo "(!) failed to download sccache v${sccache_version}. Attempting to fall back one version to retry..."; + _find_prev_version_from_git_tags sccache_version "${find_version_args[@]}"; done - # Install sccache - wget --no-hsts -q -O- \ - "https://github.com/$github_repo/releases/download/v$sccache_version/sccache-v$sccache_version-$(uname -m)-unknown-linux-musl.tar.gz" \ - | sudo tar -C /usr/bin -zf - --wildcards --strip-components=1 -x '*/sccache' \ - && sudo chmod +x /usr/bin/sccache; - echo "Installed sccache v$(sccache --version | cut -d' ' -f2) to $(which sccache)" } diff --git a/scripts/copy-common-scripts.sh b/scripts/copy-common-scripts.sh index d9814fc08..24283e7d3 100755 --- a/scripts/copy-common-scripts.sh +++ b/scripts/copy-common-scripts.sh @@ -4,3 +4,5 @@ cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."; find ./features/src -mindepth 1 -maxdepth 1 -type d -exec \ bash -c 'rm -rf {}/common && cp -ar ./features/common {}/' \; + +cp ./features/common/find-version-from-git-tags.sh ./features/src/utils/opt/devcontainer/bin/sccache/find-version-from-git-tags.sh From 71f8f39c439699c0d463c56204d589b8d7d488e4 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 5 Aug 2025 16:31:21 -0700 Subject: [PATCH 71/78] make find_version_from_git_tags filter for the next tag after a previous version --- features/common/find-version-from-git-tags.sh | 34 +++++++++++++++---- .../common/find-version-from-git-tags.sh | 34 +++++++++++++++---- .../common/find-version-from-git-tags.sh | 34 +++++++++++++++---- .../cuda/common/find-version-from-git-tags.sh | 34 +++++++++++++++---- .../gcc/common/find-version-from-git-tags.sh | 34 +++++++++++++++---- .../common/find-version-from-git-tags.sh | 34 +++++++++++++++---- .../llvm/common/find-version-from-git-tags.sh | 34 +++++++++++++++---- .../common/find-version-from-git-tags.sh | 34 +++++++++++++++---- .../common/find-version-from-git-tags.sh | 34 +++++++++++++++---- .../common/find-version-from-git-tags.sh | 34 +++++++++++++++---- .../common/find-version-from-git-tags.sh | 34 +++++++++++++++---- .../common/find-version-from-git-tags.sh | 34 +++++++++++++++---- .../common/find-version-from-git-tags.sh | 34 +++++++++++++++---- .../rust/common/find-version-from-git-tags.sh | 34 +++++++++++++++---- .../common/find-version-from-git-tags.sh | 34 +++++++++++++++---- .../ucx/common/find-version-from-git-tags.sh | 34 +++++++++++++++---- .../common/find-version-from-git-tags.sh | 34 +++++++++++++++---- .../bin/sccache/find-version-from-git-tags.sh | 34 +++++++++++++++---- .../opt/devcontainer/bin/sccache/install.sh | 16 ++++++--- 19 files changed, 497 insertions(+), 131 deletions(-) mode change 100644 => 100755 features/common/find-version-from-git-tags.sh mode change 100644 => 100755 features/src/cccl-dev/common/find-version-from-git-tags.sh mode change 100644 => 100755 features/src/cmake/common/find-version-from-git-tags.sh mode change 100644 => 100755 features/src/cuda/common/find-version-from-git-tags.sh mode change 100644 => 100755 features/src/gcc/common/find-version-from-git-tags.sh mode change 100644 => 100755 features/src/gitlab-cli/common/find-version-from-git-tags.sh mode change 100644 => 100755 features/src/llvm/common/find-version-from-git-tags.sh mode change 100644 => 100755 features/src/mambaforge/common/find-version-from-git-tags.sh mode change 100644 => 100755 features/src/ninja/common/find-version-from-git-tags.sh mode change 100644 => 100755 features/src/nvhpc/common/find-version-from-git-tags.sh mode change 100644 => 100755 features/src/oneapi/common/find-version-from-git-tags.sh mode change 100644 => 100755 features/src/openmpi/common/find-version-from-git-tags.sh mode change 100644 => 100755 features/src/rapids-build-utils/common/find-version-from-git-tags.sh mode change 100644 => 100755 features/src/rust/common/find-version-from-git-tags.sh mode change 100644 => 100755 features/src/sccache/common/find-version-from-git-tags.sh mode change 100644 => 100755 features/src/ucx/common/find-version-from-git-tags.sh mode change 100644 => 100755 features/src/utils/common/find-version-from-git-tags.sh diff --git a/features/common/find-version-from-git-tags.sh b/features/common/find-version-from-git-tags.sh old mode 100644 new mode 100755 index 9e9d649ce..b959c1310 --- a/features/common/find-version-from-git-tags.sh +++ b/features/common/find-version-from-git-tags.sh @@ -16,6 +16,8 @@ _upvar() { fi } +declare -Ag _find_version_from_git_tags_cache=(); + # Figure out correct version of a three part version number is not passed _find_version_from_git_tags() { local variable_name="$1" @@ -26,6 +28,7 @@ _find_version_from_git_tags() { local separator="${4:-"."}" local suffix="${5:-}" local last_part_optional="${6:-"false"}" + local after_version="${7:-""}" if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator="${separator//./\\.}" local last_part="" @@ -43,16 +46,33 @@ _find_version_from_git_tags() { fi fi local regex="${prefix}\\K[0-9]+${last_part}$" - local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; - if test -n "${remote_upstream_fetch:+x}"; then - git config unset --global remote.upstream.fetch - fi - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if test -n "${remote_upstream_fetch:+x}"; then - git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + + if ! test -v _find_version_from_git_tags_cache["$variable_name"]; then + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)" + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch || true + fi + readarray -t version_list < <( + git ls-remote --tags "${repository}" \ + | grep -oP "${regex}" \ + | tr -d ' ' \ + | tr "${separator}" "." \ + | sort -rV + ) + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}" + fi + _upvar _find_version_from_git_tags_cache["$variable_name"] "${version_list[*]}" + else + readarray -d' ' -t version_list <<< "${_find_version_from_git_tags_cache["$variable_name"]}" fi + local version_list="$(IFS=$'\n'; echo "${version_list[*]}")" if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then requested_version="$(head -n 1 <<< "${version_list}")" + elif test -n "${after_version:+x}"; then + set +e + requested_version="$(grep -A 1 -m 1 "${after_version}" <<< "${version_list}" | tail -n 1)" + set -e else set +e requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" diff --git a/features/src/cccl-dev/common/find-version-from-git-tags.sh b/features/src/cccl-dev/common/find-version-from-git-tags.sh old mode 100644 new mode 100755 index 9e9d649ce..b959c1310 --- a/features/src/cccl-dev/common/find-version-from-git-tags.sh +++ b/features/src/cccl-dev/common/find-version-from-git-tags.sh @@ -16,6 +16,8 @@ _upvar() { fi } +declare -Ag _find_version_from_git_tags_cache=(); + # Figure out correct version of a three part version number is not passed _find_version_from_git_tags() { local variable_name="$1" @@ -26,6 +28,7 @@ _find_version_from_git_tags() { local separator="${4:-"."}" local suffix="${5:-}" local last_part_optional="${6:-"false"}" + local after_version="${7:-""}" if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator="${separator//./\\.}" local last_part="" @@ -43,16 +46,33 @@ _find_version_from_git_tags() { fi fi local regex="${prefix}\\K[0-9]+${last_part}$" - local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; - if test -n "${remote_upstream_fetch:+x}"; then - git config unset --global remote.upstream.fetch - fi - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if test -n "${remote_upstream_fetch:+x}"; then - git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + + if ! test -v _find_version_from_git_tags_cache["$variable_name"]; then + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)" + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch || true + fi + readarray -t version_list < <( + git ls-remote --tags "${repository}" \ + | grep -oP "${regex}" \ + | tr -d ' ' \ + | tr "${separator}" "." \ + | sort -rV + ) + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}" + fi + _upvar _find_version_from_git_tags_cache["$variable_name"] "${version_list[*]}" + else + readarray -d' ' -t version_list <<< "${_find_version_from_git_tags_cache["$variable_name"]}" fi + local version_list="$(IFS=$'\n'; echo "${version_list[*]}")" if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then requested_version="$(head -n 1 <<< "${version_list}")" + elif test -n "${after_version:+x}"; then + set +e + requested_version="$(grep -A 1 -m 1 "${after_version}" <<< "${version_list}" | tail -n 1)" + set -e else set +e requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" diff --git a/features/src/cmake/common/find-version-from-git-tags.sh b/features/src/cmake/common/find-version-from-git-tags.sh old mode 100644 new mode 100755 index 9e9d649ce..b959c1310 --- a/features/src/cmake/common/find-version-from-git-tags.sh +++ b/features/src/cmake/common/find-version-from-git-tags.sh @@ -16,6 +16,8 @@ _upvar() { fi } +declare -Ag _find_version_from_git_tags_cache=(); + # Figure out correct version of a three part version number is not passed _find_version_from_git_tags() { local variable_name="$1" @@ -26,6 +28,7 @@ _find_version_from_git_tags() { local separator="${4:-"."}" local suffix="${5:-}" local last_part_optional="${6:-"false"}" + local after_version="${7:-""}" if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator="${separator//./\\.}" local last_part="" @@ -43,16 +46,33 @@ _find_version_from_git_tags() { fi fi local regex="${prefix}\\K[0-9]+${last_part}$" - local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; - if test -n "${remote_upstream_fetch:+x}"; then - git config unset --global remote.upstream.fetch - fi - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if test -n "${remote_upstream_fetch:+x}"; then - git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + + if ! test -v _find_version_from_git_tags_cache["$variable_name"]; then + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)" + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch || true + fi + readarray -t version_list < <( + git ls-remote --tags "${repository}" \ + | grep -oP "${regex}" \ + | tr -d ' ' \ + | tr "${separator}" "." \ + | sort -rV + ) + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}" + fi + _upvar _find_version_from_git_tags_cache["$variable_name"] "${version_list[*]}" + else + readarray -d' ' -t version_list <<< "${_find_version_from_git_tags_cache["$variable_name"]}" fi + local version_list="$(IFS=$'\n'; echo "${version_list[*]}")" if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then requested_version="$(head -n 1 <<< "${version_list}")" + elif test -n "${after_version:+x}"; then + set +e + requested_version="$(grep -A 1 -m 1 "${after_version}" <<< "${version_list}" | tail -n 1)" + set -e else set +e requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" diff --git a/features/src/cuda/common/find-version-from-git-tags.sh b/features/src/cuda/common/find-version-from-git-tags.sh old mode 100644 new mode 100755 index 9e9d649ce..b959c1310 --- a/features/src/cuda/common/find-version-from-git-tags.sh +++ b/features/src/cuda/common/find-version-from-git-tags.sh @@ -16,6 +16,8 @@ _upvar() { fi } +declare -Ag _find_version_from_git_tags_cache=(); + # Figure out correct version of a three part version number is not passed _find_version_from_git_tags() { local variable_name="$1" @@ -26,6 +28,7 @@ _find_version_from_git_tags() { local separator="${4:-"."}" local suffix="${5:-}" local last_part_optional="${6:-"false"}" + local after_version="${7:-""}" if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator="${separator//./\\.}" local last_part="" @@ -43,16 +46,33 @@ _find_version_from_git_tags() { fi fi local regex="${prefix}\\K[0-9]+${last_part}$" - local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; - if test -n "${remote_upstream_fetch:+x}"; then - git config unset --global remote.upstream.fetch - fi - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if test -n "${remote_upstream_fetch:+x}"; then - git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + + if ! test -v _find_version_from_git_tags_cache["$variable_name"]; then + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)" + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch || true + fi + readarray -t version_list < <( + git ls-remote --tags "${repository}" \ + | grep -oP "${regex}" \ + | tr -d ' ' \ + | tr "${separator}" "." \ + | sort -rV + ) + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}" + fi + _upvar _find_version_from_git_tags_cache["$variable_name"] "${version_list[*]}" + else + readarray -d' ' -t version_list <<< "${_find_version_from_git_tags_cache["$variable_name"]}" fi + local version_list="$(IFS=$'\n'; echo "${version_list[*]}")" if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then requested_version="$(head -n 1 <<< "${version_list}")" + elif test -n "${after_version:+x}"; then + set +e + requested_version="$(grep -A 1 -m 1 "${after_version}" <<< "${version_list}" | tail -n 1)" + set -e else set +e requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" diff --git a/features/src/gcc/common/find-version-from-git-tags.sh b/features/src/gcc/common/find-version-from-git-tags.sh old mode 100644 new mode 100755 index 9e9d649ce..b959c1310 --- a/features/src/gcc/common/find-version-from-git-tags.sh +++ b/features/src/gcc/common/find-version-from-git-tags.sh @@ -16,6 +16,8 @@ _upvar() { fi } +declare -Ag _find_version_from_git_tags_cache=(); + # Figure out correct version of a three part version number is not passed _find_version_from_git_tags() { local variable_name="$1" @@ -26,6 +28,7 @@ _find_version_from_git_tags() { local separator="${4:-"."}" local suffix="${5:-}" local last_part_optional="${6:-"false"}" + local after_version="${7:-""}" if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator="${separator//./\\.}" local last_part="" @@ -43,16 +46,33 @@ _find_version_from_git_tags() { fi fi local regex="${prefix}\\K[0-9]+${last_part}$" - local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; - if test -n "${remote_upstream_fetch:+x}"; then - git config unset --global remote.upstream.fetch - fi - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if test -n "${remote_upstream_fetch:+x}"; then - git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + + if ! test -v _find_version_from_git_tags_cache["$variable_name"]; then + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)" + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch || true + fi + readarray -t version_list < <( + git ls-remote --tags "${repository}" \ + | grep -oP "${regex}" \ + | tr -d ' ' \ + | tr "${separator}" "." \ + | sort -rV + ) + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}" + fi + _upvar _find_version_from_git_tags_cache["$variable_name"] "${version_list[*]}" + else + readarray -d' ' -t version_list <<< "${_find_version_from_git_tags_cache["$variable_name"]}" fi + local version_list="$(IFS=$'\n'; echo "${version_list[*]}")" if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then requested_version="$(head -n 1 <<< "${version_list}")" + elif test -n "${after_version:+x}"; then + set +e + requested_version="$(grep -A 1 -m 1 "${after_version}" <<< "${version_list}" | tail -n 1)" + set -e else set +e requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" diff --git a/features/src/gitlab-cli/common/find-version-from-git-tags.sh b/features/src/gitlab-cli/common/find-version-from-git-tags.sh old mode 100644 new mode 100755 index 9e9d649ce..b959c1310 --- a/features/src/gitlab-cli/common/find-version-from-git-tags.sh +++ b/features/src/gitlab-cli/common/find-version-from-git-tags.sh @@ -16,6 +16,8 @@ _upvar() { fi } +declare -Ag _find_version_from_git_tags_cache=(); + # Figure out correct version of a three part version number is not passed _find_version_from_git_tags() { local variable_name="$1" @@ -26,6 +28,7 @@ _find_version_from_git_tags() { local separator="${4:-"."}" local suffix="${5:-}" local last_part_optional="${6:-"false"}" + local after_version="${7:-""}" if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator="${separator//./\\.}" local last_part="" @@ -43,16 +46,33 @@ _find_version_from_git_tags() { fi fi local regex="${prefix}\\K[0-9]+${last_part}$" - local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; - if test -n "${remote_upstream_fetch:+x}"; then - git config unset --global remote.upstream.fetch - fi - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if test -n "${remote_upstream_fetch:+x}"; then - git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + + if ! test -v _find_version_from_git_tags_cache["$variable_name"]; then + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)" + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch || true + fi + readarray -t version_list < <( + git ls-remote --tags "${repository}" \ + | grep -oP "${regex}" \ + | tr -d ' ' \ + | tr "${separator}" "." \ + | sort -rV + ) + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}" + fi + _upvar _find_version_from_git_tags_cache["$variable_name"] "${version_list[*]}" + else + readarray -d' ' -t version_list <<< "${_find_version_from_git_tags_cache["$variable_name"]}" fi + local version_list="$(IFS=$'\n'; echo "${version_list[*]}")" if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then requested_version="$(head -n 1 <<< "${version_list}")" + elif test -n "${after_version:+x}"; then + set +e + requested_version="$(grep -A 1 -m 1 "${after_version}" <<< "${version_list}" | tail -n 1)" + set -e else set +e requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" diff --git a/features/src/llvm/common/find-version-from-git-tags.sh b/features/src/llvm/common/find-version-from-git-tags.sh old mode 100644 new mode 100755 index 9e9d649ce..b959c1310 --- a/features/src/llvm/common/find-version-from-git-tags.sh +++ b/features/src/llvm/common/find-version-from-git-tags.sh @@ -16,6 +16,8 @@ _upvar() { fi } +declare -Ag _find_version_from_git_tags_cache=(); + # Figure out correct version of a three part version number is not passed _find_version_from_git_tags() { local variable_name="$1" @@ -26,6 +28,7 @@ _find_version_from_git_tags() { local separator="${4:-"."}" local suffix="${5:-}" local last_part_optional="${6:-"false"}" + local after_version="${7:-""}" if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator="${separator//./\\.}" local last_part="" @@ -43,16 +46,33 @@ _find_version_from_git_tags() { fi fi local regex="${prefix}\\K[0-9]+${last_part}$" - local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; - if test -n "${remote_upstream_fetch:+x}"; then - git config unset --global remote.upstream.fetch - fi - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if test -n "${remote_upstream_fetch:+x}"; then - git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + + if ! test -v _find_version_from_git_tags_cache["$variable_name"]; then + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)" + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch || true + fi + readarray -t version_list < <( + git ls-remote --tags "${repository}" \ + | grep -oP "${regex}" \ + | tr -d ' ' \ + | tr "${separator}" "." \ + | sort -rV + ) + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}" + fi + _upvar _find_version_from_git_tags_cache["$variable_name"] "${version_list[*]}" + else + readarray -d' ' -t version_list <<< "${_find_version_from_git_tags_cache["$variable_name"]}" fi + local version_list="$(IFS=$'\n'; echo "${version_list[*]}")" if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then requested_version="$(head -n 1 <<< "${version_list}")" + elif test -n "${after_version:+x}"; then + set +e + requested_version="$(grep -A 1 -m 1 "${after_version}" <<< "${version_list}" | tail -n 1)" + set -e else set +e requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" diff --git a/features/src/mambaforge/common/find-version-from-git-tags.sh b/features/src/mambaforge/common/find-version-from-git-tags.sh old mode 100644 new mode 100755 index 9e9d649ce..b959c1310 --- a/features/src/mambaforge/common/find-version-from-git-tags.sh +++ b/features/src/mambaforge/common/find-version-from-git-tags.sh @@ -16,6 +16,8 @@ _upvar() { fi } +declare -Ag _find_version_from_git_tags_cache=(); + # Figure out correct version of a three part version number is not passed _find_version_from_git_tags() { local variable_name="$1" @@ -26,6 +28,7 @@ _find_version_from_git_tags() { local separator="${4:-"."}" local suffix="${5:-}" local last_part_optional="${6:-"false"}" + local after_version="${7:-""}" if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator="${separator//./\\.}" local last_part="" @@ -43,16 +46,33 @@ _find_version_from_git_tags() { fi fi local regex="${prefix}\\K[0-9]+${last_part}$" - local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; - if test -n "${remote_upstream_fetch:+x}"; then - git config unset --global remote.upstream.fetch - fi - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if test -n "${remote_upstream_fetch:+x}"; then - git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + + if ! test -v _find_version_from_git_tags_cache["$variable_name"]; then + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)" + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch || true + fi + readarray -t version_list < <( + git ls-remote --tags "${repository}" \ + | grep -oP "${regex}" \ + | tr -d ' ' \ + | tr "${separator}" "." \ + | sort -rV + ) + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}" + fi + _upvar _find_version_from_git_tags_cache["$variable_name"] "${version_list[*]}" + else + readarray -d' ' -t version_list <<< "${_find_version_from_git_tags_cache["$variable_name"]}" fi + local version_list="$(IFS=$'\n'; echo "${version_list[*]}")" if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then requested_version="$(head -n 1 <<< "${version_list}")" + elif test -n "${after_version:+x}"; then + set +e + requested_version="$(grep -A 1 -m 1 "${after_version}" <<< "${version_list}" | tail -n 1)" + set -e else set +e requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" diff --git a/features/src/ninja/common/find-version-from-git-tags.sh b/features/src/ninja/common/find-version-from-git-tags.sh old mode 100644 new mode 100755 index 9e9d649ce..b959c1310 --- a/features/src/ninja/common/find-version-from-git-tags.sh +++ b/features/src/ninja/common/find-version-from-git-tags.sh @@ -16,6 +16,8 @@ _upvar() { fi } +declare -Ag _find_version_from_git_tags_cache=(); + # Figure out correct version of a three part version number is not passed _find_version_from_git_tags() { local variable_name="$1" @@ -26,6 +28,7 @@ _find_version_from_git_tags() { local separator="${4:-"."}" local suffix="${5:-}" local last_part_optional="${6:-"false"}" + local after_version="${7:-""}" if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator="${separator//./\\.}" local last_part="" @@ -43,16 +46,33 @@ _find_version_from_git_tags() { fi fi local regex="${prefix}\\K[0-9]+${last_part}$" - local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; - if test -n "${remote_upstream_fetch:+x}"; then - git config unset --global remote.upstream.fetch - fi - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if test -n "${remote_upstream_fetch:+x}"; then - git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + + if ! test -v _find_version_from_git_tags_cache["$variable_name"]; then + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)" + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch || true + fi + readarray -t version_list < <( + git ls-remote --tags "${repository}" \ + | grep -oP "${regex}" \ + | tr -d ' ' \ + | tr "${separator}" "." \ + | sort -rV + ) + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}" + fi + _upvar _find_version_from_git_tags_cache["$variable_name"] "${version_list[*]}" + else + readarray -d' ' -t version_list <<< "${_find_version_from_git_tags_cache["$variable_name"]}" fi + local version_list="$(IFS=$'\n'; echo "${version_list[*]}")" if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then requested_version="$(head -n 1 <<< "${version_list}")" + elif test -n "${after_version:+x}"; then + set +e + requested_version="$(grep -A 1 -m 1 "${after_version}" <<< "${version_list}" | tail -n 1)" + set -e else set +e requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" diff --git a/features/src/nvhpc/common/find-version-from-git-tags.sh b/features/src/nvhpc/common/find-version-from-git-tags.sh old mode 100644 new mode 100755 index 9e9d649ce..b959c1310 --- a/features/src/nvhpc/common/find-version-from-git-tags.sh +++ b/features/src/nvhpc/common/find-version-from-git-tags.sh @@ -16,6 +16,8 @@ _upvar() { fi } +declare -Ag _find_version_from_git_tags_cache=(); + # Figure out correct version of a three part version number is not passed _find_version_from_git_tags() { local variable_name="$1" @@ -26,6 +28,7 @@ _find_version_from_git_tags() { local separator="${4:-"."}" local suffix="${5:-}" local last_part_optional="${6:-"false"}" + local after_version="${7:-""}" if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator="${separator//./\\.}" local last_part="" @@ -43,16 +46,33 @@ _find_version_from_git_tags() { fi fi local regex="${prefix}\\K[0-9]+${last_part}$" - local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; - if test -n "${remote_upstream_fetch:+x}"; then - git config unset --global remote.upstream.fetch - fi - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if test -n "${remote_upstream_fetch:+x}"; then - git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + + if ! test -v _find_version_from_git_tags_cache["$variable_name"]; then + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)" + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch || true + fi + readarray -t version_list < <( + git ls-remote --tags "${repository}" \ + | grep -oP "${regex}" \ + | tr -d ' ' \ + | tr "${separator}" "." \ + | sort -rV + ) + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}" + fi + _upvar _find_version_from_git_tags_cache["$variable_name"] "${version_list[*]}" + else + readarray -d' ' -t version_list <<< "${_find_version_from_git_tags_cache["$variable_name"]}" fi + local version_list="$(IFS=$'\n'; echo "${version_list[*]}")" if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then requested_version="$(head -n 1 <<< "${version_list}")" + elif test -n "${after_version:+x}"; then + set +e + requested_version="$(grep -A 1 -m 1 "${after_version}" <<< "${version_list}" | tail -n 1)" + set -e else set +e requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" diff --git a/features/src/oneapi/common/find-version-from-git-tags.sh b/features/src/oneapi/common/find-version-from-git-tags.sh old mode 100644 new mode 100755 index 9e9d649ce..b959c1310 --- a/features/src/oneapi/common/find-version-from-git-tags.sh +++ b/features/src/oneapi/common/find-version-from-git-tags.sh @@ -16,6 +16,8 @@ _upvar() { fi } +declare -Ag _find_version_from_git_tags_cache=(); + # Figure out correct version of a three part version number is not passed _find_version_from_git_tags() { local variable_name="$1" @@ -26,6 +28,7 @@ _find_version_from_git_tags() { local separator="${4:-"."}" local suffix="${5:-}" local last_part_optional="${6:-"false"}" + local after_version="${7:-""}" if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator="${separator//./\\.}" local last_part="" @@ -43,16 +46,33 @@ _find_version_from_git_tags() { fi fi local regex="${prefix}\\K[0-9]+${last_part}$" - local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; - if test -n "${remote_upstream_fetch:+x}"; then - git config unset --global remote.upstream.fetch - fi - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if test -n "${remote_upstream_fetch:+x}"; then - git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + + if ! test -v _find_version_from_git_tags_cache["$variable_name"]; then + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)" + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch || true + fi + readarray -t version_list < <( + git ls-remote --tags "${repository}" \ + | grep -oP "${regex}" \ + | tr -d ' ' \ + | tr "${separator}" "." \ + | sort -rV + ) + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}" + fi + _upvar _find_version_from_git_tags_cache["$variable_name"] "${version_list[*]}" + else + readarray -d' ' -t version_list <<< "${_find_version_from_git_tags_cache["$variable_name"]}" fi + local version_list="$(IFS=$'\n'; echo "${version_list[*]}")" if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then requested_version="$(head -n 1 <<< "${version_list}")" + elif test -n "${after_version:+x}"; then + set +e + requested_version="$(grep -A 1 -m 1 "${after_version}" <<< "${version_list}" | tail -n 1)" + set -e else set +e requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" diff --git a/features/src/openmpi/common/find-version-from-git-tags.sh b/features/src/openmpi/common/find-version-from-git-tags.sh old mode 100644 new mode 100755 index 9e9d649ce..b959c1310 --- a/features/src/openmpi/common/find-version-from-git-tags.sh +++ b/features/src/openmpi/common/find-version-from-git-tags.sh @@ -16,6 +16,8 @@ _upvar() { fi } +declare -Ag _find_version_from_git_tags_cache=(); + # Figure out correct version of a three part version number is not passed _find_version_from_git_tags() { local variable_name="$1" @@ -26,6 +28,7 @@ _find_version_from_git_tags() { local separator="${4:-"."}" local suffix="${5:-}" local last_part_optional="${6:-"false"}" + local after_version="${7:-""}" if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator="${separator//./\\.}" local last_part="" @@ -43,16 +46,33 @@ _find_version_from_git_tags() { fi fi local regex="${prefix}\\K[0-9]+${last_part}$" - local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; - if test -n "${remote_upstream_fetch:+x}"; then - git config unset --global remote.upstream.fetch - fi - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if test -n "${remote_upstream_fetch:+x}"; then - git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + + if ! test -v _find_version_from_git_tags_cache["$variable_name"]; then + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)" + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch || true + fi + readarray -t version_list < <( + git ls-remote --tags "${repository}" \ + | grep -oP "${regex}" \ + | tr -d ' ' \ + | tr "${separator}" "." \ + | sort -rV + ) + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}" + fi + _upvar _find_version_from_git_tags_cache["$variable_name"] "${version_list[*]}" + else + readarray -d' ' -t version_list <<< "${_find_version_from_git_tags_cache["$variable_name"]}" fi + local version_list="$(IFS=$'\n'; echo "${version_list[*]}")" if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then requested_version="$(head -n 1 <<< "${version_list}")" + elif test -n "${after_version:+x}"; then + set +e + requested_version="$(grep -A 1 -m 1 "${after_version}" <<< "${version_list}" | tail -n 1)" + set -e else set +e requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" diff --git a/features/src/rapids-build-utils/common/find-version-from-git-tags.sh b/features/src/rapids-build-utils/common/find-version-from-git-tags.sh old mode 100644 new mode 100755 index 9e9d649ce..b959c1310 --- a/features/src/rapids-build-utils/common/find-version-from-git-tags.sh +++ b/features/src/rapids-build-utils/common/find-version-from-git-tags.sh @@ -16,6 +16,8 @@ _upvar() { fi } +declare -Ag _find_version_from_git_tags_cache=(); + # Figure out correct version of a three part version number is not passed _find_version_from_git_tags() { local variable_name="$1" @@ -26,6 +28,7 @@ _find_version_from_git_tags() { local separator="${4:-"."}" local suffix="${5:-}" local last_part_optional="${6:-"false"}" + local after_version="${7:-""}" if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator="${separator//./\\.}" local last_part="" @@ -43,16 +46,33 @@ _find_version_from_git_tags() { fi fi local regex="${prefix}\\K[0-9]+${last_part}$" - local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; - if test -n "${remote_upstream_fetch:+x}"; then - git config unset --global remote.upstream.fetch - fi - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if test -n "${remote_upstream_fetch:+x}"; then - git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + + if ! test -v _find_version_from_git_tags_cache["$variable_name"]; then + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)" + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch || true + fi + readarray -t version_list < <( + git ls-remote --tags "${repository}" \ + | grep -oP "${regex}" \ + | tr -d ' ' \ + | tr "${separator}" "." \ + | sort -rV + ) + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}" + fi + _upvar _find_version_from_git_tags_cache["$variable_name"] "${version_list[*]}" + else + readarray -d' ' -t version_list <<< "${_find_version_from_git_tags_cache["$variable_name"]}" fi + local version_list="$(IFS=$'\n'; echo "${version_list[*]}")" if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then requested_version="$(head -n 1 <<< "${version_list}")" + elif test -n "${after_version:+x}"; then + set +e + requested_version="$(grep -A 1 -m 1 "${after_version}" <<< "${version_list}" | tail -n 1)" + set -e else set +e requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" diff --git a/features/src/rust/common/find-version-from-git-tags.sh b/features/src/rust/common/find-version-from-git-tags.sh old mode 100644 new mode 100755 index 9e9d649ce..b959c1310 --- a/features/src/rust/common/find-version-from-git-tags.sh +++ b/features/src/rust/common/find-version-from-git-tags.sh @@ -16,6 +16,8 @@ _upvar() { fi } +declare -Ag _find_version_from_git_tags_cache=(); + # Figure out correct version of a three part version number is not passed _find_version_from_git_tags() { local variable_name="$1" @@ -26,6 +28,7 @@ _find_version_from_git_tags() { local separator="${4:-"."}" local suffix="${5:-}" local last_part_optional="${6:-"false"}" + local after_version="${7:-""}" if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator="${separator//./\\.}" local last_part="" @@ -43,16 +46,33 @@ _find_version_from_git_tags() { fi fi local regex="${prefix}\\K[0-9]+${last_part}$" - local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; - if test -n "${remote_upstream_fetch:+x}"; then - git config unset --global remote.upstream.fetch - fi - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if test -n "${remote_upstream_fetch:+x}"; then - git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + + if ! test -v _find_version_from_git_tags_cache["$variable_name"]; then + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)" + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch || true + fi + readarray -t version_list < <( + git ls-remote --tags "${repository}" \ + | grep -oP "${regex}" \ + | tr -d ' ' \ + | tr "${separator}" "." \ + | sort -rV + ) + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}" + fi + _upvar _find_version_from_git_tags_cache["$variable_name"] "${version_list[*]}" + else + readarray -d' ' -t version_list <<< "${_find_version_from_git_tags_cache["$variable_name"]}" fi + local version_list="$(IFS=$'\n'; echo "${version_list[*]}")" if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then requested_version="$(head -n 1 <<< "${version_list}")" + elif test -n "${after_version:+x}"; then + set +e + requested_version="$(grep -A 1 -m 1 "${after_version}" <<< "${version_list}" | tail -n 1)" + set -e else set +e requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" diff --git a/features/src/sccache/common/find-version-from-git-tags.sh b/features/src/sccache/common/find-version-from-git-tags.sh old mode 100644 new mode 100755 index 9e9d649ce..b959c1310 --- a/features/src/sccache/common/find-version-from-git-tags.sh +++ b/features/src/sccache/common/find-version-from-git-tags.sh @@ -16,6 +16,8 @@ _upvar() { fi } +declare -Ag _find_version_from_git_tags_cache=(); + # Figure out correct version of a three part version number is not passed _find_version_from_git_tags() { local variable_name="$1" @@ -26,6 +28,7 @@ _find_version_from_git_tags() { local separator="${4:-"."}" local suffix="${5:-}" local last_part_optional="${6:-"false"}" + local after_version="${7:-""}" if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator="${separator//./\\.}" local last_part="" @@ -43,16 +46,33 @@ _find_version_from_git_tags() { fi fi local regex="${prefix}\\K[0-9]+${last_part}$" - local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; - if test -n "${remote_upstream_fetch:+x}"; then - git config unset --global remote.upstream.fetch - fi - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if test -n "${remote_upstream_fetch:+x}"; then - git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + + if ! test -v _find_version_from_git_tags_cache["$variable_name"]; then + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)" + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch || true + fi + readarray -t version_list < <( + git ls-remote --tags "${repository}" \ + | grep -oP "${regex}" \ + | tr -d ' ' \ + | tr "${separator}" "." \ + | sort -rV + ) + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}" + fi + _upvar _find_version_from_git_tags_cache["$variable_name"] "${version_list[*]}" + else + readarray -d' ' -t version_list <<< "${_find_version_from_git_tags_cache["$variable_name"]}" fi + local version_list="$(IFS=$'\n'; echo "${version_list[*]}")" if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then requested_version="$(head -n 1 <<< "${version_list}")" + elif test -n "${after_version:+x}"; then + set +e + requested_version="$(grep -A 1 -m 1 "${after_version}" <<< "${version_list}" | tail -n 1)" + set -e else set +e requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" diff --git a/features/src/ucx/common/find-version-from-git-tags.sh b/features/src/ucx/common/find-version-from-git-tags.sh old mode 100644 new mode 100755 index 9e9d649ce..b959c1310 --- a/features/src/ucx/common/find-version-from-git-tags.sh +++ b/features/src/ucx/common/find-version-from-git-tags.sh @@ -16,6 +16,8 @@ _upvar() { fi } +declare -Ag _find_version_from_git_tags_cache=(); + # Figure out correct version of a three part version number is not passed _find_version_from_git_tags() { local variable_name="$1" @@ -26,6 +28,7 @@ _find_version_from_git_tags() { local separator="${4:-"."}" local suffix="${5:-}" local last_part_optional="${6:-"false"}" + local after_version="${7:-""}" if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator="${separator//./\\.}" local last_part="" @@ -43,16 +46,33 @@ _find_version_from_git_tags() { fi fi local regex="${prefix}\\K[0-9]+${last_part}$" - local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; - if test -n "${remote_upstream_fetch:+x}"; then - git config unset --global remote.upstream.fetch - fi - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if test -n "${remote_upstream_fetch:+x}"; then - git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + + if ! test -v _find_version_from_git_tags_cache["$variable_name"]; then + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)" + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch || true + fi + readarray -t version_list < <( + git ls-remote --tags "${repository}" \ + | grep -oP "${regex}" \ + | tr -d ' ' \ + | tr "${separator}" "." \ + | sort -rV + ) + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}" + fi + _upvar _find_version_from_git_tags_cache["$variable_name"] "${version_list[*]}" + else + readarray -d' ' -t version_list <<< "${_find_version_from_git_tags_cache["$variable_name"]}" fi + local version_list="$(IFS=$'\n'; echo "${version_list[*]}")" if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then requested_version="$(head -n 1 <<< "${version_list}")" + elif test -n "${after_version:+x}"; then + set +e + requested_version="$(grep -A 1 -m 1 "${after_version}" <<< "${version_list}" | tail -n 1)" + set -e else set +e requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" diff --git a/features/src/utils/common/find-version-from-git-tags.sh b/features/src/utils/common/find-version-from-git-tags.sh old mode 100644 new mode 100755 index 9e9d649ce..b959c1310 --- a/features/src/utils/common/find-version-from-git-tags.sh +++ b/features/src/utils/common/find-version-from-git-tags.sh @@ -16,6 +16,8 @@ _upvar() { fi } +declare -Ag _find_version_from_git_tags_cache=(); + # Figure out correct version of a three part version number is not passed _find_version_from_git_tags() { local variable_name="$1" @@ -26,6 +28,7 @@ _find_version_from_git_tags() { local separator="${4:-"."}" local suffix="${5:-}" local last_part_optional="${6:-"false"}" + local after_version="${7:-""}" if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator="${separator//./\\.}" local last_part="" @@ -43,16 +46,33 @@ _find_version_from_git_tags() { fi fi local regex="${prefix}\\K[0-9]+${last_part}$" - local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; - if test -n "${remote_upstream_fetch:+x}"; then - git config unset --global remote.upstream.fetch - fi - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if test -n "${remote_upstream_fetch:+x}"; then - git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + + if ! test -v _find_version_from_git_tags_cache["$variable_name"]; then + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)" + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch || true + fi + readarray -t version_list < <( + git ls-remote --tags "${repository}" \ + | grep -oP "${regex}" \ + | tr -d ' ' \ + | tr "${separator}" "." \ + | sort -rV + ) + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}" + fi + _upvar _find_version_from_git_tags_cache["$variable_name"] "${version_list[*]}" + else + readarray -d' ' -t version_list <<< "${_find_version_from_git_tags_cache["$variable_name"]}" fi + local version_list="$(IFS=$'\n'; echo "${version_list[*]}")" if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then requested_version="$(head -n 1 <<< "${version_list}")" + elif test -n "${after_version:+x}"; then + set +e + requested_version="$(grep -A 1 -m 1 "${after_version}" <<< "${version_list}" | tail -n 1)" + set -e else set +e requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" diff --git a/features/src/utils/opt/devcontainer/bin/sccache/find-version-from-git-tags.sh b/features/src/utils/opt/devcontainer/bin/sccache/find-version-from-git-tags.sh index 9e9d649ce..b959c1310 100644 --- a/features/src/utils/opt/devcontainer/bin/sccache/find-version-from-git-tags.sh +++ b/features/src/utils/opt/devcontainer/bin/sccache/find-version-from-git-tags.sh @@ -16,6 +16,8 @@ _upvar() { fi } +declare -Ag _find_version_from_git_tags_cache=(); + # Figure out correct version of a three part version number is not passed _find_version_from_git_tags() { local variable_name="$1" @@ -26,6 +28,7 @@ _find_version_from_git_tags() { local separator="${4:-"."}" local suffix="${5:-}" local last_part_optional="${6:-"false"}" + local after_version="${7:-""}" if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator="${separator//./\\.}" local last_part="" @@ -43,16 +46,33 @@ _find_version_from_git_tags() { fi fi local regex="${prefix}\\K[0-9]+${last_part}$" - local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"; - if test -n "${remote_upstream_fetch:+x}"; then - git config unset --global remote.upstream.fetch - fi - local -r version_list="$(git ls-remote --tags "${repository}" | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" - if test -n "${remote_upstream_fetch:+x}"; then - git config set --global remote.upstream.fetch "${remote_upstream_fetch}"; + + if ! test -v _find_version_from_git_tags_cache["$variable_name"]; then + local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)" + if test -n "${remote_upstream_fetch:+x}"; then + git config unset --global remote.upstream.fetch || true + fi + readarray -t version_list < <( + git ls-remote --tags "${repository}" \ + | grep -oP "${regex}" \ + | tr -d ' ' \ + | tr "${separator}" "." \ + | sort -rV + ) + if test -n "${remote_upstream_fetch:+x}"; then + git config set --global remote.upstream.fetch "${remote_upstream_fetch}" + fi + _upvar _find_version_from_git_tags_cache["$variable_name"] "${version_list[*]}" + else + readarray -d' ' -t version_list <<< "${_find_version_from_git_tags_cache["$variable_name"]}" fi + local version_list="$(IFS=$'\n'; echo "${version_list[*]}")" if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then requested_version="$(head -n 1 <<< "${version_list}")" + elif test -n "${after_version:+x}"; then + set +e + requested_version="$(grep -A 1 -m 1 "${after_version}" <<< "${version_list}" | tail -n 1)" + set -e else set +e requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")" diff --git a/features/src/utils/opt/devcontainer/bin/sccache/install.sh b/features/src/utils/opt/devcontainer/bin/sccache/install.sh index 439c92177..a29a8376f 100755 --- a/features/src/utils/opt/devcontainer/bin/sccache/install.sh +++ b/features/src/utils/opt/devcontainer/bin/sccache/install.sh @@ -42,14 +42,20 @@ _install_sccache() { _find_version_from_git_tags sccache_version "${find_version_args[@]}"; - while ! wget --no-hsts -q -O- "https://github.com/$github_repo/releases/download/v$sccache_version/sccache-v$sccache_version-$(uname -m)-unknown-linux-musl.tar.gz" \ - | sudo tar -C /usr/bin -zf - --wildcards --strip-components=1 -x '*/sccache' \ - && sudo chmod +x /usr/bin/sccache; do + while test -n "${sccache_version:+x}" && \ + ! wget --no-hsts -q -O- "https://github.com/$github_repo/releases/download/v$sccache_version/sccache-v$sccache_version-$(uname -m)-unknown-linux-musl.tar.gz" \ + | sudo tar -C /usr/bin -zf - --wildcards --strip-components=1 -x '*/sccache' 2>/dev/null \ + && sudo chmod +x /usr/bin/sccache; do echo "(!) failed to download sccache v${sccache_version}. Attempting to fall back one version to retry..."; - _find_prev_version_from_git_tags sccache_version "${find_version_args[@]}"; + _find_version_from_git_tags sccache_version "${find_version_args[@]}" "${sccache_version}"; done - echo "Installed sccache v$(sccache --version | cut -d' ' -f2) to $(which sccache)" + if test -n "${sccache_version:+x}"; then + echo "Installed sccache v$(sccache --version | cut -d' ' -f2) to $(which sccache)"; + else + echo "(!) failed to download sccache" >&2; + return 1 + fi } _install_sccache "$@" <&0; From 319129a0648ac943ce1f1561ac97bcf301ad8865 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 6 Aug 2025 23:12:31 -0700 Subject: [PATCH 72/78] fix shell init script for nvhpc25.7 --- .../nvhpc/etc/profile.d/{nvhpc.sh => nvhpc25.5.sh} | 0 features/src/nvhpc/etc/profile.d/nvhpc25.7.sh | 7 +++++++ features/src/nvhpc/install.sh | 12 +++++++++--- 3 files changed, 16 insertions(+), 3 deletions(-) rename features/src/nvhpc/etc/profile.d/{nvhpc.sh => nvhpc25.5.sh} (100%) create mode 100755 features/src/nvhpc/etc/profile.d/nvhpc25.7.sh diff --git a/features/src/nvhpc/etc/profile.d/nvhpc.sh b/features/src/nvhpc/etc/profile.d/nvhpc25.5.sh similarity index 100% rename from features/src/nvhpc/etc/profile.d/nvhpc.sh rename to features/src/nvhpc/etc/profile.d/nvhpc25.5.sh diff --git a/features/src/nvhpc/etc/profile.d/nvhpc25.7.sh b/features/src/nvhpc/etc/profile.d/nvhpc25.7.sh new file mode 100755 index 000000000..8ff37a726 --- /dev/null +++ b/features/src/nvhpc/etc/profile.d/nvhpc25.7.sh @@ -0,0 +1,7 @@ +if ! command -v module 2>&1 | grep -q function; then + . /etc/profile.d/lmod._sh; +fi + +if [ -n "${PATH##*"${NVHPC_ROOT}/compilers/bin"*}" ]; then + export PATH="${NVHPC_ROOT}/compilers/bin:$PATH"; +fi diff --git a/features/src/nvhpc/install.sh b/features/src/nvhpc/install.sh index 47a173fb2..81542dd68 100644 --- a/features/src/nvhpc/install.sh +++ b/features/src/nvhpc/install.sh @@ -80,11 +80,17 @@ vars_+=('$NVHPC_CUDA_HOME'); vars_+=('$NVHPC_MODULEFILE_DIRS'); printf -v vars_ '%s,' "${vars_[@]}"; +if [[ 25.7 > "$NVHPC_VERSION" ]]; then + nvhpcrc="etc/profile.d/nvhpc25.5.sh" +else + nvhpcrc="etc/profile.d/nvhpc25.7.sh" +fi + # export envvars in bashrc files -append_to_etc_bashrc "$(cat <(cat .bashrc | envsubst "${vars_%,}") etc/profile.d/nvhpc.sh)"; -append_to_all_bashrcs "$(cat <(cat .bashrc | envsubst "${vars_%,}") etc/profile.d/nvhpc.sh)"; +append_to_etc_bashrc "$(cat <(cat .bashrc | envsubst "${vars_%,}") "$nvhpcrc")"; +append_to_all_bashrcs "$(cat <(cat .bashrc | envsubst "${vars_%,}") "$nvhpcrc")"; # export envvars in /etc/profile.d -add_etc_profile_d_script nvhpc "$(cat <(cat .bashrc | envsubst "${vars_%,}") etc/profile.d/nvhpc.sh)"; +add_etc_profile_d_script nvhpc "$(cat <(cat .bashrc | envsubst "${vars_%,}") "$nvhpcrc")"; # Clean up # rm -rf /tmp/*; From f215b073c49bcd7b7d29cff952bb96300f46c922 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 8 Aug 2025 16:38:00 -0700 Subject: [PATCH 73/78] Revert "fix shell init script for nvhpc25.7" This reverts commit 319129a0648ac943ce1f1561ac97bcf301ad8865. --- .../nvhpc/etc/profile.d/{nvhpc25.5.sh => nvhpc.sh} | 0 features/src/nvhpc/etc/profile.d/nvhpc25.7.sh | 7 ------- features/src/nvhpc/install.sh | 12 +++--------- 3 files changed, 3 insertions(+), 16 deletions(-) rename features/src/nvhpc/etc/profile.d/{nvhpc25.5.sh => nvhpc.sh} (100%) delete mode 100755 features/src/nvhpc/etc/profile.d/nvhpc25.7.sh diff --git a/features/src/nvhpc/etc/profile.d/nvhpc25.5.sh b/features/src/nvhpc/etc/profile.d/nvhpc.sh similarity index 100% rename from features/src/nvhpc/etc/profile.d/nvhpc25.5.sh rename to features/src/nvhpc/etc/profile.d/nvhpc.sh diff --git a/features/src/nvhpc/etc/profile.d/nvhpc25.7.sh b/features/src/nvhpc/etc/profile.d/nvhpc25.7.sh deleted file mode 100755 index 8ff37a726..000000000 --- a/features/src/nvhpc/etc/profile.d/nvhpc25.7.sh +++ /dev/null @@ -1,7 +0,0 @@ -if ! command -v module 2>&1 | grep -q function; then - . /etc/profile.d/lmod._sh; -fi - -if [ -n "${PATH##*"${NVHPC_ROOT}/compilers/bin"*}" ]; then - export PATH="${NVHPC_ROOT}/compilers/bin:$PATH"; -fi diff --git a/features/src/nvhpc/install.sh b/features/src/nvhpc/install.sh index 81542dd68..47a173fb2 100644 --- a/features/src/nvhpc/install.sh +++ b/features/src/nvhpc/install.sh @@ -80,17 +80,11 @@ vars_+=('$NVHPC_CUDA_HOME'); vars_+=('$NVHPC_MODULEFILE_DIRS'); printf -v vars_ '%s,' "${vars_[@]}"; -if [[ 25.7 > "$NVHPC_VERSION" ]]; then - nvhpcrc="etc/profile.d/nvhpc25.5.sh" -else - nvhpcrc="etc/profile.d/nvhpc25.7.sh" -fi - # export envvars in bashrc files -append_to_etc_bashrc "$(cat <(cat .bashrc | envsubst "${vars_%,}") "$nvhpcrc")"; -append_to_all_bashrcs "$(cat <(cat .bashrc | envsubst "${vars_%,}") "$nvhpcrc")"; +append_to_etc_bashrc "$(cat <(cat .bashrc | envsubst "${vars_%,}") etc/profile.d/nvhpc.sh)"; +append_to_all_bashrcs "$(cat <(cat .bashrc | envsubst "${vars_%,}") etc/profile.d/nvhpc.sh)"; # export envvars in /etc/profile.d -add_etc_profile_d_script nvhpc "$(cat <(cat .bashrc | envsubst "${vars_%,}") "$nvhpcrc")"; +add_etc_profile_d_script nvhpc "$(cat <(cat .bashrc | envsubst "${vars_%,}") etc/profile.d/nvhpc.sh)"; # Clean up # rm -rf /tmp/*; From 7d88444627d75397b25dedc283ee8e8d7555ef31 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 8 Aug 2025 18:51:41 -0700 Subject: [PATCH 74/78] fix shell init script for nvhpc25.7 --- features/src/nvhpc/etc/profile.d/nvhpc.sh | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/features/src/nvhpc/etc/profile.d/nvhpc.sh b/features/src/nvhpc/etc/profile.d/nvhpc.sh index 6edae13c0..b49be54e3 100755 --- a/features/src/nvhpc/etc/profile.d/nvhpc.sh +++ b/features/src/nvhpc/etc/profile.d/nvhpc.sh @@ -3,11 +3,18 @@ if ! command -v module 2>&1 | grep -q function; then fi if [ -n "${PATH##*"${NVHPC_ROOT}/compilers/bin"*}" ]; then - for i in ${!NVHPC_MODULEFILE_DIRS[@]}; do - module use "${NVHPC_MODULEFILE_DIRS[$i]}"; + for NVHPC_MODULEFILES_DIR in "${NVHPC_MODULEFILE_DIRS[@]}"; do + if [ -n "${MODULEPATH##*"${NVHPC_MODULEFILES_DIR}"*}" ]; then + module use -a "${NVHPC_MODULEFILES_DIR}"; + fi done - module try-load "nvhpc-nompi/${NVHPC_VERSION}" >/dev/null 2>&1; - module try-load "nvhpc-nompi" >/dev/null 2>&1; - module try-load "hpcx-mt" >/dev/null 2>&1; - module try-load "hpcx" >/dev/null 2>&1; + unset NVHPC_MODULEFILES_DIR + for NVHPC_MODULE_NAME in "nvhpc-hpcx/${NVHPC_VERSION}"; do + if ! module list "${NVHPC_MODULE_NAME}" 2>&1 | grep -q 'None found.'; then + if ! module list 2>&1 | grep -q "${NVHPC_MODULE_NAME}"; then + module try-load "${NVHPC_MODULE_NAME}" >/dev/null 2>&1; + fi + fi + done + unset NVHPC_MODULE_NAME; fi From 77fe8c1c3939d97fa1bda405bcd651a1153bae02 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 11 Aug 2025 09:52:42 -0700 Subject: [PATCH 75/78] switch back to shared-workflows branch-25.10 --- .github/workflows/build-all-rapids-repos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index ae0a5c075..2579ce17b 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -27,7 +27,7 @@ jobs: if: needs.check-event.outputs.ok == 'true' needs: check-event secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fea/preprocessor-caching + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.10 permissions: actions: read packages: read From dccfebc557b11f0855f041da599a7ede429eece4 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sat, 2 Aug 2025 11:42:02 -0700 Subject: [PATCH 76/78] tee build times out to /var/log --- .../bin/tmpl/cpp.build.tmpl.sh | 4 +-- .../bin/tmpl/python.build.wheel.tmpl.sh | 5 ++-- .../bin/tmpl/python.install.tmpl.sh | 27 +++++++++---------- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh index 1c1309053..630ec214e 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh @@ -50,9 +50,9 @@ build_${CPP_LIB}_cpp() { local -a cmake_build_args="($(rapids-select-cmake-build-args ${n_jobs:+-j${n_jobs}} "${OPTS[@]}"))"; cmake \ --build "${CPP_SRC}/${BIN_DIR}" \ - "${cmake_build_args[@]}" ; + "${cmake_build_args[@]}" 2>&1 ; { set +x; } 2>/dev/null; echo -n "lib${CPP_LIB} build time:"; - ) 2>&1; + ) 2> >(tee -a /var/log/devcontainer-utils/build-${CPP_LIB}-cpp-time.log >&2); } build_${CPP_LIB}_cpp "$@" <&0; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh index 973f24bf2..79569e362 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh @@ -115,10 +115,9 @@ EOF SKBUILD_CMAKE_BUILD_TYPE="${build_type}" \ CMAKE_BUILD_PARALLEL_LEVEL="${n_jobs}" \ NVCC_APPEND_FLAGS="${nvcc_append_flags}" \ - python -m pip wheel "${pip_args[@]}" \ - ; + python -m pip wheel "${pip_args[@]}" 2>&1; { set +x; } 2>/dev/null; echo -n "${PY_LIB} wheel build time:"; - ) 2>&1; + ) 2> >(tee -a /var/log/devcontainer-utils/install-${PY_LIB}-python-time.log >&2); } build_${PY_LIB}_python_wheel "$@" <&0; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh index 167c44fdb..bd7a462d9 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh @@ -122,21 +122,20 @@ EOF local build_type="$(rapids-select-cmake-build-type "${cmake_args_[@]}" || echo "Release")"; local nvcc_append_flags="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }-t=${n_arch}"; - CUDAFLAGS="${cudaflags}" \ - CMAKE_GENERATOR="${G:-Ninja}" \ - PARALLEL_LEVEL="${n_jobs}" \ - CMAKE_ARGS="${cmake_args[*]@Q}" \ - SKBUILD_BUILD_OPTIONS="${ninja_args[*]}" \ - SKBUILD_BUILD_VERBOSE="${v:+True}" \ - SKBUILD_LOGGING_LEVEL="${v:+INFO}" \ - SKBUILD_INSTALL_STRIP="${strip:+True}" \ - SKBUILD_CMAKE_BUILD_TYPE="${build_type}" \ - CMAKE_BUILD_PARALLEL_LEVEL="${n_jobs}" \ - NVCC_APPEND_FLAGS="${nvcc_append_flags}" \ - python -m pip install "${pip_args[@]}" \ - ; + CUDAFLAGS="${cudaflags}" \ + CMAKE_GENERATOR="${G:-Ninja}" \ + PARALLEL_LEVEL="${n_jobs}" \ + CMAKE_ARGS="${cmake_args[*]@Q}" \ + SKBUILD_BUILD_OPTIONS="${ninja_args[*]}" \ + SKBUILD_BUILD_VERBOSE="${v:+True}" \ + SKBUILD_LOGGING_LEVEL="${v:+INFO}" \ + SKBUILD_INSTALL_STRIP="${strip:+True}" \ + SKBUILD_CMAKE_BUILD_TYPE="${build_type}" \ + CMAKE_BUILD_PARALLEL_LEVEL="${n_jobs}" \ + NVCC_APPEND_FLAGS="${nvcc_append_flags}" \ + python -m pip install "${pip_args[@]}" 2>&1; { set +x; } 2>/dev/null; echo -n "${PY_LIB} install time:"; - ) 2>&1; + ) 2> >(tee -a /var/log/devcontainer-utils/install-${PY_LIB}-python-time.log >&2); } install_${PY_LIB}_python "$@" <&0; From e1b8de287e86615727aafb7e8dede0e5d5183eea Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 11 Aug 2025 09:57:14 -0700 Subject: [PATCH 77/78] print build times at the end --- .github/workflows/build-all-rapids-repos.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 2579ce17b..eb63c81fd 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -78,3 +78,7 @@ jobs: # Print cache and dist stats sccache --show-adv-stats; + + # Print build times + find /var/log/devcontainer-utils/ -type f -name 'build-*-time.log' -print0 \ + | xargs -0 -n1 grep -H real | sed 's/real\t/ /g' || : # Nonfatal if not found From 768745bec2f016f1baea22707463b4083b302c19 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 11 Aug 2025 10:20:31 -0700 Subject: [PATCH 78/78] remove SCCACHE_S3_KEY_PREFIX --- .devcontainer/rapids.Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile index 545c8c1b5..5d95a11d1 100644 --- a/.devcontainer/rapids.Dockerfile +++ b/.devcontainer/rapids.Dockerfile @@ -54,7 +54,6 @@ ENV SCCACHE_REGION="us-east-2" ENV SCCACHE_BUCKET="rapids-sccache-devs" # 2hr (1 minute longer than sccache-dist request timeout) ENV SCCACHE_IDLE_TIMEOUT=7200 -ENV SCCACHE_S3_KEY_PREFIX=rapids-test-sccache-dist ### # sccache-dist configuration