feat(ccache): add opt-in in-cluster Redis remote backend

.github/workflows/multi_arch_build_portable_linux.yml

@@ -61,7 +61,7 @@ on:
         type: string
         default: ""
       cache_type:
-        description: "Compiler cache type (sccache, ccache, or none)"
+        description: "Compiler cache type (sccache, ccache, ccache-redis, or none)"
         type: string
         default: "sccache"
 
@@ -228,6 +228,7 @@ jobs:
       repository: ${{ inputs.repository }}
       ref: ${{ inputs.ref }}
       external_repo_config: ${{ inputs.external_repo_config }}
+      cache_type: ${{ inputs.cache_type }}
     permissions:
       contents: read
       id-token: write

.github/workflows/multi_arch_build_portable_linux_artifacts.yml

@@ -70,7 +70,7 @@ on:
         type: string
         default: ""
       cache_type:
-        description: "Compiler cache type (sccache, ccache, or none)"
+        description: "Compiler cache type (sccache, ccache, ccache-redis, or none)"
         type: string
         default: "sccache"
 
@@ -130,8 +130,17 @@ jobs:
 
       - name: Setup ccache
         run: |
+          # ccache-redis opts into the in-cluster Redis remote (github-oss-redis
+          # preset); all other cache types keep the release-type mapping
+          # (bazel-remote presets). --config-preset and --release-type are
+          # mutually exclusive, so pass exactly one.
+          if [ "${{ inputs.cache_type }}" = "ccache-redis" ]; then
+            CCACHE_PRESET_ARGS="--config-preset github-oss-redis"
+          else
+            CCACHE_PRESET_ARGS="--release-type ${{ inputs.release_type }}"
+          fi
           ./build_tools/setup_ccache.py \
-            --release-type "${{ inputs.release_type }}" \
+            ${CCACHE_PRESET_ARGS} \
             --dir "$(dirname $CCACHE_CONFIGPATH)" \
             --local-path "$CACHE_DIR/ccache"
 
@@ -186,7 +195,7 @@ jobs:
           COMPILER_LAUNCHER_ARGS=""
           if [ "${{ inputs.cache_type }}" = "sccache" ]; then
             COMPILER_LAUNCHER_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache"
-          elif [ "${{ inputs.cache_type }}" = "ccache" ]; then
+          elif [ "${{ inputs.cache_type }}" = "ccache" ] || [ "${{ inputs.cache_type }}" = "ccache-redis" ]; then
             COMPILER_LAUNCHER_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
           fi
           # Disable unity/jumbo builds so sccache can cache individual compilations.
@@ -231,7 +240,7 @@ jobs:
           if [ "${{ inputs.cache_type }}" = "sccache" ]; then
             echo "Sccache Stats:"
             sccache --show-stats || true
-          elif [ "${{ inputs.cache_type }}" = "ccache" ]; then
+          elif [ "${{ inputs.cache_type }}" = "ccache" ] || [ "${{ inputs.cache_type }}" = "ccache-redis" ]; then
             echo "CCache Stats:"
             ccache -s -v
           fi

.github/workflows/multi_arch_ci.yml

@@ -50,6 +50,15 @@ on:
         type: boolean
         default: true
         description: "Build PyTorch wheels"
+      cache_type:
+        type: choice
+        default: "sccache"
+        description: "Compiler cache for Linux builds. Default sccache; ccache-redis opts into the in-cluster Redis backend."
+        options:
+          - sccache
+          - ccache
+          - ccache-redis
+          - none
   pull_request:
     types:
       - labeled
@@ -103,6 +112,8 @@ jobs:
       rocm_rpm_package_version: ${{ needs.setup.outputs.rocm_rpm_package_version }}
       test_type: ${{ needs.setup.outputs.test_type }}
       changed_projects: ${{ inputs.changed_projects || '' }}
+      # Opt-in only via manual dispatch; PR/push default to sccache.
+      cache_type: ${{ inputs.cache_type || 'sccache' }}
     permissions:
       contents: read
       id-token: write

.github/workflows/multi_arch_ci_linux.yml

@@ -48,6 +48,10 @@ on:
         description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref."
         type: string
         default: ""
+      cache_type:
+        description: "Compiler cache type (sccache, ccache, ccache-redis, or none)"
+        type: string
+        default: "sccache"
 
 permissions:
   contents: read
@@ -112,6 +116,7 @@ jobs:
       external_repo_config: ${{ inputs.external_repo_config }}
       repository: ${{ inputs.repository }}
       ref: ${{ inputs.ref }}
+      cache_type: ${{ inputs.cache_type }}
     permissions:
       contents: read
       id-token: write

build_tools/setup_ccache.py

@@ -25,6 +25,7 @@
 """
 
 import argparse
+import os
 from pathlib import Path
 import platform
 import sys
@@ -40,6 +41,16 @@
 CACHE_SRV_DEV = "http://bazelremote-svc.bazelremote-ns.svc.cluster.local:8080|layout=bazel|connect-timeout=50"
 CACHE_SRV_REL = "http://bazelremote-svc-rel.bazelremote-ns.svc.cluster.local:8080|layout=bazel|connect-timeout=50"
 
+# Redis (Valkey) remote cache: an in-cluster, ClusterIP-only service in the
+# prod build cluster (therock-runners-prod). No authentication, matching the
+# bazel-remote posture (relies on cluster network isolation). The endpoint is
+# overridable via CCACHE_REDIS_ENDPOINT so it is not hardcoded per environment.
+CACHE_SRV_REDIS_ENDPOINT = os.environ.get(
+    "CCACHE_REDIS_ENDPOINT",
+    "redis-ccache-svc.redis-ccache-ns.svc.cluster.local:6379",
+)
+CACHE_SRV_REDIS = f"redis://{CACHE_SRV_REDIS_ENDPOINT}|connect-timeout=50"
+
 # Bump this version when making hash-affecting config changes (sloppiness,
 # compiler_check, etc.) to logically isolate new cache entries from stale
 # ones on the shared remote cache server.
@@ -68,6 +79,14 @@
         "max_size": "10G",
         "namespace": f"therock-{CCACHE_NAMESPACE_VERSION}",
     },
+    # In-cluster Redis (Valkey) remote cache. Opt-in via the CI cache_type
+    # input; the default remains sccache. max_size below applies to the local
+    # cache tier that fronts the remote Redis store.
+    "github-oss-redis": {
+        "remote_storage": CACHE_SRV_REDIS,
+        "max_size": "10G",
+        "namespace": f"therock-{CCACHE_NAMESPACE_VERSION}",
+    },
 }
 
 

build_tools/setup_sccache_rocm.py

@@ -157,7 +157,9 @@ def main():
                 if key == "HIP_CLANG_LAUNCHER":
                     f.write(f"{key}={value}\n")
         if "HIP_CLANG_LAUNCHER" in env:
-            print(f"Wrote HIP_CLANG_LAUNCHER={env['HIP_CLANG_LAUNCHER']} to $GITHUB_ENV")
+            print(
+                f"Wrote HIP_CLANG_LAUNCHER={env['HIP_CLANG_LAUNCHER']} to $GITHUB_ENV"
+            )
     else:
         print("Configure a ROCm build with:")
         for key, value in env.items():

docs/development/ccache_troubleshooting.md

@@ -15,8 +15,11 @@ cache with similar goals. We are evaluating both ccache and sccache for
 use across ROCm and downstream frameworks like PyTorch (see
 `build_tools/setup_sccache_rocm.py`). The goal is to standardize on a
 cache setup that works well across our full ecosystem of projects using
-shared infrastructure (e.g., cache servers). This page focuses on ccache,
-which is the current default for TheRock CI.
+shared infrastructure (e.g., cache servers). This page focuses on ccache.
+
+Note: multi-arch CI currently defaults to **sccache** (S3-backed); ccache is
+used for local/developer builds, as the bazel-remote-backed fallback, and as
+the opt-in in-cluster Redis backend described below.
 
 ### Key files
 
@@ -42,6 +45,31 @@ remote cache, accessed via ccache's `remote_storage` option with
 Both servers are on the Kubernetes cluster, accessible without
 authentication from any pod in the cluster.
 
+### In-cluster Redis backend (opt-in)
+
+The `github-oss-redis` preset points ccache's `remote_storage` at an
+in-cluster Redis (Valkey) service in the prod build cluster
+(`therock-runners-prod`):
+
+| Preset             | Server                                                    | Used by                          |
+| ------------------ | --------------------------------------------------------- | -------------------------------- |
+| `github-oss-redis` | `redis-ccache-svc.redis-ccache-ns.svc.cluster.local:6379` | Opt-in `cache_type=ccache-redis` |
+
+Like bazel-remote, it is a `ClusterIP` service with **no authentication**,
+relying on cluster network isolation. The endpoint can be overridden with the
+`CCACHE_REDIS_ENDPOINT` environment variable. It is deployed from
+`ROCm/TheRock-Infra` (`helm/redis-ccache`, `deploy-redis-ccache-prod.yaml`).
+
+Select it from a manual `Multi-Arch CI` run via the `cache_type` input
+(`ccache-redis`); PR/push builds keep the default (`sccache`). Since the
+service is in-cluster only, builds must run on the AWS in-cluster pool
+(`aws-linux-scale-rocm-prod`). Windows/sanitizer builds on Azure cannot reach
+it until they move into the same AWS cluster.
+
+Caveat: ccache's built-in Redis backend has no TLS and is **deprecated for
+removal in ccache 4.14/4.15**. This backend works with the pinned 4.11.2 but
+has a shelf-life; revisit if/when ccache is upgraded.
+
 ### Namespace version
 
 `CCACHE_NAMESPACE_VERSION` in `setup_ccache.py` controls the cache