From acb541da510719a3cbad2d7697579aab2ac2d0f4 Mon Sep 17 00:00:00 2001 From: Subodh Dubey Date: Fri, 10 Jul 2026 11:18:13 +0000 Subject: [PATCH 1/2] feat(ccache): add opt-in in-cluster Redis remote backend Adds a github-oss-redis preset to setup_ccache.py and threads an opt-in cache_type=ccache-redis through the multi-arch Linux CI workflows. Default remains sccache. Redis is deployed in-cluster via ROCm/TheRock-Infra. Co-authored-by: Cursor --- .../multi_arch_build_portable_linux.yml | 3 +- ...ti_arch_build_portable_linux_artifacts.yml | 17 +++++++--- .github/workflows/multi_arch_ci.yml | 11 +++++++ .github/workflows/multi_arch_ci_linux.yml | 5 +++ build_tools/setup_ccache.py | 19 +++++++++++ docs/development/ccache_troubleshooting.md | 32 +++++++++++++++++-- 6 files changed, 80 insertions(+), 7 deletions(-) diff --git a/.github/workflows/multi_arch_build_portable_linux.yml b/.github/workflows/multi_arch_build_portable_linux.yml index 17b2aaf11ce..02586f9fd59 100644 --- a/.github/workflows/multi_arch_build_portable_linux.yml +++ b/.github/workflows/multi_arch_build_portable_linux.yml @@ -61,7 +61,7 @@ on: type: string default: "" cache_type: - description: "Compiler cache type (sccache, ccache, or none)" + description: "Compiler cache type (sccache, ccache, ccache-redis, or none)" type: string default: "sccache" @@ -228,6 +228,7 @@ jobs: repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} external_repo_config: ${{ inputs.external_repo_config }} + cache_type: ${{ inputs.cache_type }} permissions: contents: read id-token: write diff --git a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml index 2b081db0abe..0349011538c 100644 --- a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml +++ b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml @@ -70,7 +70,7 @@ on: type: string default: "" cache_type: - description: "Compiler cache type (sccache, ccache, or none)" + description: "Compiler cache type (sccache, ccache, ccache-redis, or none)" type: string default: "sccache" @@ -130,8 +130,17 @@ jobs: - name: Setup ccache run: | + # ccache-redis opts into the in-cluster Redis remote (github-oss-redis + # preset); all other cache types keep the release-type mapping + # (bazel-remote presets). --config-preset and --release-type are + # mutually exclusive, so pass exactly one. + if [ "${{ inputs.cache_type }}" = "ccache-redis" ]; then + CCACHE_PRESET_ARGS="--config-preset github-oss-redis" + else + CCACHE_PRESET_ARGS="--release-type ${{ inputs.release_type }}" + fi ./build_tools/setup_ccache.py \ - --release-type "${{ inputs.release_type }}" \ + ${CCACHE_PRESET_ARGS} \ --dir "$(dirname $CCACHE_CONFIGPATH)" \ --local-path "$CACHE_DIR/ccache" @@ -186,7 +195,7 @@ jobs: COMPILER_LAUNCHER_ARGS="" if [ "${{ inputs.cache_type }}" = "sccache" ]; then COMPILER_LAUNCHER_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache" - elif [ "${{ inputs.cache_type }}" = "ccache" ]; then + elif [ "${{ inputs.cache_type }}" = "ccache" ] || [ "${{ inputs.cache_type }}" = "ccache-redis" ]; then COMPILER_LAUNCHER_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" fi # Disable unity/jumbo builds so sccache can cache individual compilations. @@ -231,7 +240,7 @@ jobs: if [ "${{ inputs.cache_type }}" = "sccache" ]; then echo "Sccache Stats:" sccache --show-stats || true - elif [ "${{ inputs.cache_type }}" = "ccache" ]; then + elif [ "${{ inputs.cache_type }}" = "ccache" ] || [ "${{ inputs.cache_type }}" = "ccache-redis" ]; then echo "CCache Stats:" ccache -s -v fi diff --git a/.github/workflows/multi_arch_ci.yml b/.github/workflows/multi_arch_ci.yml index 0f284772705..baf6e0cd4e0 100644 --- a/.github/workflows/multi_arch_ci.yml +++ b/.github/workflows/multi_arch_ci.yml @@ -50,6 +50,15 @@ on: type: boolean default: true description: "Build PyTorch wheels" + cache_type: + type: choice + default: "sccache" + description: "Compiler cache for Linux builds. Default sccache; ccache-redis opts into the in-cluster Redis backend." + options: + - sccache + - ccache + - ccache-redis + - none pull_request: types: - labeled @@ -103,6 +112,8 @@ jobs: rocm_rpm_package_version: ${{ needs.setup.outputs.rocm_rpm_package_version }} test_type: ${{ needs.setup.outputs.test_type }} changed_projects: ${{ inputs.changed_projects || '' }} + # Opt-in only via manual dispatch; PR/push default to sccache. + cache_type: ${{ inputs.cache_type || 'sccache' }} permissions: contents: read id-token: write diff --git a/.github/workflows/multi_arch_ci_linux.yml b/.github/workflows/multi_arch_ci_linux.yml index f9ff5ba0bc4..f47478e8dbe 100644 --- a/.github/workflows/multi_arch_ci_linux.yml +++ b/.github/workflows/multi_arch_ci_linux.yml @@ -48,6 +48,10 @@ on: description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." type: string default: "" + cache_type: + description: "Compiler cache type (sccache, ccache, ccache-redis, or none)" + type: string + default: "sccache" permissions: contents: read @@ -112,6 +116,7 @@ jobs: external_repo_config: ${{ inputs.external_repo_config }} repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} + cache_type: ${{ inputs.cache_type }} permissions: contents: read id-token: write diff --git a/build_tools/setup_ccache.py b/build_tools/setup_ccache.py index f65b89fb5f3..e305e483030 100755 --- a/build_tools/setup_ccache.py +++ b/build_tools/setup_ccache.py @@ -25,6 +25,7 @@ """ import argparse +import os from pathlib import Path import platform import sys @@ -40,6 +41,16 @@ CACHE_SRV_DEV = "http://bazelremote-svc.bazelremote-ns.svc.cluster.local:8080|layout=bazel|connect-timeout=50" CACHE_SRV_REL = "http://bazelremote-svc-rel.bazelremote-ns.svc.cluster.local:8080|layout=bazel|connect-timeout=50" +# Redis (Valkey) remote cache: an in-cluster, ClusterIP-only service in the +# prod build cluster (therock-runners-prod). No authentication, matching the +# bazel-remote posture (relies on cluster network isolation). The endpoint is +# overridable via CCACHE_REDIS_ENDPOINT so it is not hardcoded per environment. +CACHE_SRV_REDIS_ENDPOINT = os.environ.get( + "CCACHE_REDIS_ENDPOINT", + "redis-ccache-svc.redis-ccache-ns.svc.cluster.local:6379", +) +CACHE_SRV_REDIS = f"redis://{CACHE_SRV_REDIS_ENDPOINT}|connect-timeout=50" + # Bump this version when making hash-affecting config changes (sloppiness, # compiler_check, etc.) to logically isolate new cache entries from stale # ones on the shared remote cache server. @@ -68,6 +79,14 @@ "max_size": "10G", "namespace": f"therock-{CCACHE_NAMESPACE_VERSION}", }, + # In-cluster Redis (Valkey) remote cache. Opt-in via the CI cache_type + # input; the default remains sccache. max_size below applies to the local + # cache tier that fronts the remote Redis store. + "github-oss-redis": { + "remote_storage": CACHE_SRV_REDIS, + "max_size": "10G", + "namespace": f"therock-{CCACHE_NAMESPACE_VERSION}", + }, } diff --git a/docs/development/ccache_troubleshooting.md b/docs/development/ccache_troubleshooting.md index 42630d73699..90159cc3b8b 100644 --- a/docs/development/ccache_troubleshooting.md +++ b/docs/development/ccache_troubleshooting.md @@ -15,8 +15,11 @@ cache with similar goals. We are evaluating both ccache and sccache for use across ROCm and downstream frameworks like PyTorch (see `build_tools/setup_sccache_rocm.py`). The goal is to standardize on a cache setup that works well across our full ecosystem of projects using -shared infrastructure (e.g., cache servers). This page focuses on ccache, -which is the current default for TheRock CI. +shared infrastructure (e.g., cache servers). This page focuses on ccache. + +Note: multi-arch CI currently defaults to **sccache** (S3-backed); ccache is +used for local/developer builds, as the bazel-remote-backed fallback, and as +the opt-in in-cluster Redis backend described below. ### Key files @@ -42,6 +45,31 @@ remote cache, accessed via ccache's `remote_storage` option with Both servers are on the Kubernetes cluster, accessible without authentication from any pod in the cluster. +### In-cluster Redis backend (opt-in) + +The `github-oss-redis` preset points ccache's `remote_storage` at an +in-cluster Redis (Valkey) service in the prod build cluster +(`therock-runners-prod`): + +| Preset | Server | Used by | +| ------------------ | ------------------------------------------------------------ | -------------------------------- | +| `github-oss-redis` | `redis-ccache-svc.redis-ccache-ns.svc.cluster.local:6379` | Opt-in `cache_type=ccache-redis` | + +Like bazel-remote, it is a `ClusterIP` service with **no authentication**, +relying on cluster network isolation. The endpoint can be overridden with the +`CCACHE_REDIS_ENDPOINT` environment variable. It is deployed from +`ROCm/TheRock-Infra` (`helm/redis-ccache`, `deploy-redis-ccache-prod.yaml`). + +Select it from a manual `Multi-Arch CI` run via the `cache_type` input +(`ccache-redis`); PR/push builds keep the default (`sccache`). Since the +service is in-cluster only, builds must run on the AWS in-cluster pool +(`aws-linux-scale-rocm-prod`). Windows/sanitizer builds on Azure cannot reach +it until they move into the same AWS cluster. + +Caveat: ccache's built-in Redis backend has no TLS and is **deprecated for +removal in ccache 4.14/4.15**. This backend works with the pinned 4.11.2 but +has a shelf-life; revisit if/when ccache is upgraded. + ### Namespace version `CCACHE_NAMESPACE_VERSION` in `setup_ccache.py` controls the cache From e8dcf6c8b68ffab2ff20b61dc9559a7c4df26931 Mon Sep 17 00:00:00 2001 From: Subodh Dubey Date: Fri, 10 Jul 2026 12:29:08 +0000 Subject: [PATCH 2/2] style: satisfy pre-commit (black + mdformat) - mdformat: fix table whitespace in ccache_troubleshooting.md - black: wrap long line in setup_sccache_rocm.py (pre-existing, flagged by --all-files) --- build_tools/setup_sccache_rocm.py | 4 +++- docs/development/ccache_troubleshooting.md | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/build_tools/setup_sccache_rocm.py b/build_tools/setup_sccache_rocm.py index f98696fff2e..b559ceb4566 100644 --- a/build_tools/setup_sccache_rocm.py +++ b/build_tools/setup_sccache_rocm.py @@ -157,7 +157,9 @@ def main(): if key == "HIP_CLANG_LAUNCHER": f.write(f"{key}={value}\n") if "HIP_CLANG_LAUNCHER" in env: - print(f"Wrote HIP_CLANG_LAUNCHER={env['HIP_CLANG_LAUNCHER']} to $GITHUB_ENV") + print( + f"Wrote HIP_CLANG_LAUNCHER={env['HIP_CLANG_LAUNCHER']} to $GITHUB_ENV" + ) else: print("Configure a ROCm build with:") for key, value in env.items(): diff --git a/docs/development/ccache_troubleshooting.md b/docs/development/ccache_troubleshooting.md index 90159cc3b8b..9415549e4fd 100644 --- a/docs/development/ccache_troubleshooting.md +++ b/docs/development/ccache_troubleshooting.md @@ -51,9 +51,9 @@ The `github-oss-redis` preset points ccache's `remote_storage` at an in-cluster Redis (Valkey) service in the prod build cluster (`therock-runners-prod`): -| Preset | Server | Used by | -| ------------------ | ------------------------------------------------------------ | -------------------------------- | -| `github-oss-redis` | `redis-ccache-svc.redis-ccache-ns.svc.cluster.local:6379` | Opt-in `cache_type=ccache-redis` | +| Preset | Server | Used by | +| ------------------ | --------------------------------------------------------- | -------------------------------- | +| `github-oss-redis` | `redis-ccache-svc.redis-ccache-ns.svc.cluster.local:6379` | Opt-in `cache_type=ccache-redis` | Like bazel-remote, it is a `ClusterIP` service with **no authentication**, relying on cluster network isolation. The endpoint can be overridden with the