Use the sccache-dist build cluster in CI

.devcontainer/cuda12.9-conda/devcontainer.json

@@ -11,7 +11,9 @@
   "runArgs": [
     "--rm",
     "--name",
-    "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.10-cuda12.9-conda"
+    "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.10-cuda12.9-conda",
+    "--ulimit",
+    "nofile=500000"
   ],
   "hostRequirements": {"gpu": "optional"},
   "features": {

.devcontainer/cuda12.9-pip/devcontainer.json

@@ -11,7 +11,9 @@
   "runArgs": [
     "--rm",
     "--name",
-    "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.10-cuda12.9-pip"
+    "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.10-cuda12.9-pip",
+    "--ulimit",
+    "nofile=500000"
   ],
   "hostRequirements": {"gpu": "optional"},
   "features": {

.devcontainer/rapids.Dockerfile

@@ -30,6 +30,8 @@ ENV DEFAULT_CONDA_ENV=rapids
 
 FROM ${PYTHON_PACKAGE_MANAGER}-base
 
+ARG TARGETARCH
+
 ARG CUDA
 ENV CUDAARCHS="RAPIDS"
 ENV CUDA_VERSION="${CUDA_VERSION:-${CUDA}}"
@@ -41,11 +43,36 @@ ENV PYTHONSAFEPATH="1"
 ENV PYTHONUNBUFFERED="1"
 ENV PYTHONDONTWRITEBYTECODE="1"
 
+ENV HISTFILE="/home/coder/.cache/._bash_history"
+ENV LIBCUDF_KERNEL_CACHE_PATH="/home/coder/cudf/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest/jitify_cache"
+
+###
+# sccache configuration
+###
+ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs"
 ENV SCCACHE_REGION="us-east-2"
 ENV SCCACHE_BUCKET="rapids-sccache-devs"
-ENV SCCACHE_IDLE_TIMEOUT=900
-ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs"
+# 2hr (1 minute longer than sccache-dist request timeout)
+ENV SCCACHE_IDLE_TIMEOUT=7200
 
-ENV HISTFILE="/home/coder/.cache/._bash_history"
+###
+# sccache-dist configuration
+###
+# Enable sccache-dist by default
+ENV DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1
+# Compile locally if max retries exceeded
+ENV SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=true
+# Retry transient errors 4 times (for a total of 5 attempts)
+ENV SCCACHE_DIST_MAX_RETRIES=4
+ENV SCCACHE_DIST_CONNECT_TIMEOUT=30
+ENV SCCACHE_DIST_CONNECTION_POOL=false
+# 1hr 59min (to accommodate debug builds)
+ENV SCCACHE_DIST_REQUEST_TIMEOUT=7140
+ENV SCCACHE_DIST_KEEPALIVE_ENABLED=true
+ENV SCCACHE_DIST_KEEPALIVE_INTERVAL=20
+ENV SCCACHE_DIST_KEEPALIVE_TIMEOUT=600
+ENV SCCACHE_DIST_URL="https://${TARGETARCH}.linux.sccache.rapids.nvidia.com"
 
-ENV LIBCUDF_KERNEL_CACHE_PATH="/home/coder/cudf/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest/jitify_cache"
+# Build as much in parallel as possible
+ENV INFER_NUM_DEVICE_ARCHITECTURES=1
+ENV MAX_DEVICE_OBJ_TO_COMPILE_IN_PARALLEL=20

.github/actions/build-and-test-feature/action.yml

@@ -8,6 +8,7 @@ inputs:
   aws_role_arn: {type: string, defaut: '', required: false}
   rw_sccache_bucket: {type: string, defaut: '', required: false}
   rw_sccache_region: {type: string, defaut: '', required: false}
+  sccache_dist_scheduler_url: {type: string, defaut: '', required: false}
 
 runs:
   using: composite
@@ -32,3 +33,4 @@ runs:
         aws_role_arn: "${{ inputs.aws_role_arn }}"
         rw_sccache_bucket: "${{ inputs.rw_sccache_bucket }}"
         rw_sccache_region: "${{ inputs.rw_sccache_region }}"
+        sccache_dist_scheduler_url: "${{ inputs.sccache_dist_scheduler_url }}"

.github/workflows/build-all-rapids-repos.yml

@@ -37,26 +37,48 @@ jobs:
     with:
       arch: '["amd64", "arm64"]'
       cuda: '["12.9"]'
-      node_type: cpu32
+      node_type: cpu8
       extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY
+      rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN
       timeout-minutes: 720
-      # Prevent the sccache server from shutting down
+      # 1. Infinitely retry transient errors
+      # 2. Enable debug logging to track cache misses
+      # 3. Never fallback to locally compiling
+      # 4. Use RAPIDS_AUX_SECRET_1 as the sccache-dist auth token
       env: |
         CONDA_ENV_CREATE_QUIET=1
-        SCCACHE_IDLE_TIMEOUT=0
+        SCCACHE_DIST_MAX_RETRIES=inf
+        SCCACHE_SERVER_LOG=sccache=debug
+        SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false
+        SCCACHE_DIST_AUTH_TOKEN_VAR=RAPIDS_AUX_SECRET_1
       build_command: |
-        SCCACHE_NO_DAEMON=1 sccache --stop-server
-        sccache -z;
+        # Verify sccache cache location
         sccache --show-adv-stats;
-        clone-all -j$(nproc) -v -q --clone-upstream --single-branch --shallow-submodules;
-        build-all                        \
-          -Wno-dev                       \
-          -j$(nproc --ignore=1)          \
-          -DBUILD_SHARED_LIBS=ON         \
-          -DBUILD_TESTS=ON               \
-          -DBUILD_BENCHMARKS=ON          \
-          -DBUILD_PRIMS_BENCH=ON         \
-          -DRAFT_COMPILE_LIBRARY=ON      \
-          -DBUILD_CUGRAPH_MG_TESTS=ON    \
-          ;
+
+        # Clone all the repos
+        clone-all -j$(nproc) -v -q --clone-upstream --depth 1 --single-branch --shallow-submodules;
+
+        # Configure all the C++ libs
+        time configure-all            \
+          -j0                         \
+          -GNinja                     \
+          -Wno-dev                    \
+          -DBUILD_TESTS=ON            \
+          -DBUILD_BENCHMARKS=ON       \
+          -DBUILD_PRIMS_BENCH=ON      \
+          -DBUILD_SHARED_LIBS=ON      \
+          -DRAFT_COMPILE_LIBRARY=ON   \
+          -DBUILD_CUGRAPH_MG_TESTS=ON ;
+
+        # Build all the C++ libs
+        time build-all-cpp -j0;
+
+        # Build all the Python libs
+        time build-all-python -j0;
+
+        # Print cache and dist stats
         sccache --show-adv-stats;
+
+        # Print build times
+        find /var/log/devcontainer-utils/ -type f -name 'build-*-time.log' -print0 \
+          | xargs -0 -n1 grep -H real | sed 's/real\t/ /g' || : # Nonfatal if not found

.github/workflows/build-and-test-feature.yml

@@ -40,3 +40,4 @@ jobs:
           aws_role_arn: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs' || '' }}"
           rw_sccache_bucket: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'rapids-sccache-devs' || '' }}"
           rw_sccache_region: "${{ vars.AWS_REGION }}"
+          sccache_dist_scheduler_url: "sccache.rapids.nvidia.com"

features/common/etc/skel/.config/clangd/config.yaml

@@ -59,3 +59,5 @@ CompileFlags:
     - "--expt-relaxed-constexpr"
     - "-forward-unknown-to-host-compiler"
     - "-Werror=cross-execution-space-call"
+    - "--compress-mode*"
+    - "-static-global-template-stub*"

features/common/find-version-from-git-tags.sh

@@ -0,0 +1,129 @@
+#! /usr/bin/env bash
+
+# Assign variable one scope above the caller
+# Usage: local "$1" && _upvar $1 "value(s)"
+# Param: $1  Variable name to assign value to
+# Param: $*  Value(s) to assign.  If multiple values, an array is
+#            assigned, otherwise a single value is assigned.
+# See: http://fvue.nl/wiki/Bash:_Passing_variables_by_reference
+_upvar() {
+    if unset -v "$1"; then
+        if (( $# == 2 )); then
+            eval $1=\"\$2\";
+        else
+            eval $1=\(\"\${@:2}\"\);
+        fi;
+    fi
+}
+
+declare -Ag _find_version_from_git_tags_cache=();
+
+# Figure out correct version of a three part version number is not passed
+_find_version_from_git_tags() {
+    local variable_name="$1"
+    local requested_version="${!variable_name}"
+    if [ "${requested_version}" = "none" ]; then return; fi
+    local repository="$2"
+    local prefix="${3:-"tags/v"}"
+    local separator="${4:-"."}"
+    local suffix="${5:-}"
+    local last_part_optional="${6:-"false"}"
+    local after_version="${7:-""}"
+    if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then
+        local escaped_separator="${separator//./\\.}"
+        local last_part=""
+        if [ "${last_part_optional}" = "true" ]; then
+            last_part+="(${escaped_separator}[0-9]+)?"
+            last_part+="(${escaped_separator}[0-9]+)?"
+            if [ -n "${suffix:+x}" ]; then
+                last_part+="(${suffix})?"
+            fi
+        else
+            last_part+="${escaped_separator}[0-9]+"
+            last_part+="${escaped_separator}[0-9]+"
+            if [ -n "${suffix:+x}" ]; then
+                last_part+="(${suffix})"
+            fi
+        fi
+        local regex="${prefix}\\K[0-9]+${last_part}$"
+
+        if ! test -v _find_version_from_git_tags_cache["$variable_name"]; then
+            local remote_upstream_fetch="$(git --no-pager config get remote.upstream.fetch)"
+            if test -n "${remote_upstream_fetch:+x}"; then
+                git config unset --global remote.upstream.fetch || true
+            fi
+            readarray -t version_list < <(
+                git ls-remote --tags "${repository}" \
+              | grep -oP "${regex}"                  \
+              | tr -d ' '                            \
+              | tr "${separator}" "."                \
+              | sort -rV
+            )
+            if test -n "${remote_upstream_fetch:+x}"; then
+                git config set --global remote.upstream.fetch "${remote_upstream_fetch}"
+            fi
+            _upvar _find_version_from_git_tags_cache["$variable_name"] "${version_list[*]}"
+        else
+            readarray -d' ' -t version_list <<< "${_find_version_from_git_tags_cache["$variable_name"]}"
+        fi
+        local version_list="$(IFS=$'\n'; echo "${version_list[*]}")"
+        if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then
+            requested_version="$(head -n 1 <<< "${version_list}")"
+        elif test -n "${after_version:+x}"; then
+            set +e
+            requested_version="$(grep -A 1 -m 1 "${after_version}" <<< "${version_list}" | tail -n 1)"
+            set -e
+        else
+            set +e
+            requested_version="$(grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)" <<< "${version_list}")"
+            set -e
+        fi
+    fi
+    if [ ! -n "${requested_version:+x}" ] || ! grep "^${requested_version//./\\.}$" <<< "${version_list}" > /dev/null 2>&1; then
+        echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2
+        return 1
+    fi
+    _upvar "${variable_name}" "${requested_version}"
+    echo "${variable_name}=${requested_version}"
+}
+
+# Use semver logic to decrement a version number then look for the closest match
+_find_prev_version_from_git_tags() {
+    local variable_name="$1"
+    local current_version="${!variable_name}"
+    local repository="$2"
+    # Normally a "v" is used before the version number, but support alternate cases
+    local prefix="${3:-"tags/v"}"
+    # Some repositories use "_" instead of "." for version number part separation, support that
+    local separator="${4:-"."}"
+    # Some repositories may have tags that include a suffix (e.g. actions/node-versions)
+    local version_suffix_regex="${5:-}"
+    # Some tools release versions that omit the last digit (e.g. go)
+    local last_part_optional="${6:-"false"}"
+    # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios.
+    set +e
+    local major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')"
+    local minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')"
+    local breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')"
+
+    if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then
+        ((major=major-1))
+        _upvar "${variable_name}" "${major}"
+        # Look for latest version from previous major release
+        _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}"
+    # Handle situations like Go's odd version pattern where "0" releases omit the last part
+    elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then
+        ((minor=minor-1))
+        _upvar "${variable_name}" "${major}.${minor}"
+        # Look for latest version from previous minor release
+        _find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${version_suffix_regex}" "${last_part_optional}"
+    else
+        ((breakfix=breakfix-1))
+        if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then
+            _upvar "${variable_name}" "${major}.${minor}"
+        else
+            _upvar "${variable_name}" "${major}.${minor}.${breakfix}"
+        fi
+    fi
+    set -e
+}