From 618d303b78d4a2f57fc7e8ff1e151f2b964bd194 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 4 Nov 2025 20:34:27 -0700 Subject: [PATCH 1/4] Use Artifacts instead of Registry for moving CI Image We would like to eventually move to a fork-based development model. However, PR's from fork's are unable to push images to the GHCR registry (they could push something bad). So instead, this changes CI to use artifacts instead to pass around the container image. Surprisingly, this actually slightly reduces the time it takes to load & start the Docker container. --- .github/workflows/build_ais.yml | 55 ++++++++++++++++++++++------ .github/workflows/hipfile-nvidia.yml | 10 ++++- 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build_ais.yml b/.github/workflows/build_ais.yml index a7650964..798e5d62 100644 --- a/.github/workflows/build_ais.yml +++ b/.github/workflows/build_ais.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Set AIS CI image environment variables run: | - echo "AIS_CI_DEV_IMAGE=${{ env.AIS_DOCKER_REGISTRY }}/${{ env.AIS_CI_IMAGE_NAME }}_dev:$(echo ${{ github.ref }} \ + echo "AIS_CI_DEV_IMAGE=${{ env.AIS_CI_IMAGE_NAME }}_dev:$(echo ${{ github.ref }} \ | sed 's|[^a-zA-Z0-9]|-|g')" >> "$GITHUB_ENV" echo "AIS_CI_LATEST_IMAGE=${{ env.AIS_DOCKER_REGISTRY }}/${{ env.AIS_CI_IMAGE_NAME }}:latest" >> "$GITHUB_ENV" echo "AIS_PR_NUMBER=$(echo ${{ github.ref }} | sed 's|[^0-9]||g')" >> "$GITHUB_ENV" @@ -47,11 +47,18 @@ jobs: ${{ env.AIS_CI_IMAGE_NAME }} Development Image for AIS CI using branch \ ${{ github.head_ref }} for PR #${AIS_PR_NUMBER}. \ PR URL: ${{ env.AIS_PR_BASE_URL }}/${AIS_PR_NUMBER}" \ - --cache-to=type=registry,ref="${{ env.AIS_DOCKER_REGISTRY }}/${{ env.AIS_CI_IMAGE_NAME }}_dev:cache" \ --cache-from=type=registry,ref="${{ env.AIS_DOCKER_REGISTRY }}/${{ env.AIS_CI_IMAGE_NAME }}_dev:cache" \ - --push \ - -t ${AIS_CI_DEV_IMAGE} \ + --output=type=docker,dest=${GITHUB_WORKSPACE}/${{ env.AIS_CI_IMAGE_NAME}}_dev.tar \ + -t ${{ env.AIS_CI_IMAGE_NAME }}_dev \ ${GITHUB_WORKSPACE} + - name: Upload AIS CI image as an artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ env.AIS_CI_IMAGE_NAME }}_dev.tar + path: ${{ github.workspace }}/${{ env.AIS_CI_IMAGE_NAME }}_dev.tar + compression-level: 0 + if-no-files-found: error + retention-days: 1 compile_on_AMD: runs-on: [ubuntu-24.04] needs: build_AIS_image @@ -71,6 +78,13 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Download AIS CI image artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.AIS_CI_IMAGE_NAME }}_dev.tar + path: ${{ github.workspace }} + - name: Load AIS CI image into Docker + run: docker load --input ${GITHUB_WORKSPACE}/${{ env.AIS_CI_IMAGE_NAME }}_dev.tar # Detach the container and run separate commands to it. # Thus we can make separate explicit steps in the Github CI # as if we were able to parameterize the container image in the first place. @@ -79,11 +93,10 @@ jobs: docker run \ -dt \ --rm \ - --pull always \ -v ${GITHUB_WORKSPACE}:/mnt/ais:ro \ -v ${{ env.AIS_MOUNT_PATH }}:/mnt/ais-fs \ --name ${AIS_CONTAINER_NAME} \ - ${AIS_CI_DEV_IMAGE} + ${{ env.AIS_CI_IMAGE_NAME }}_dev - name: Make copy of the code repository and create build directories # Single quotes necessary to ensure string/command substitutions happen # in the container and not on the host. @@ -181,16 +194,22 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Download AIS CI image artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.AIS_CI_IMAGE_NAME }}_dev.tar + path: ${{ github.workspace }} + - name: Load AIS CI image into Docker + run: docker load --input ${GITHUB_WORKSPACE}/${{ env.AIS_CI_IMAGE_NAME }}_dev.tar - name: Starting Docker Container run: | docker run \ -dt \ --rm \ - --pull always \ -v ${GITHUB_WORKSPACE}:/mnt/ais:ro \ -v ${{ env.AIS_MOUNT_PATH }}:/mnt/ais-fs \ --name ${AIS_CONTAINER_NAME} \ - ${AIS_CI_DEV_IMAGE} + ${{ env.AIS_CI_IMAGE_NAME }}_dev - name: Make copy of the code repository and create build directories run: | docker exec \ @@ -246,6 +265,13 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Download AIS CI image artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.AIS_CI_IMAGE_NAME }}_dev.tar + path: ${{ github.workspace }} + - name: Load AIS CI image into Docker + run: docker load --input ${GITHUB_WORKSPACE}/${{ env.AIS_CI_IMAGE_NAME }}_dev.tar # Detach the container and run separate commands to it. # Thus we can make separate explicit steps in the Github CI # as if we were able to parameterize the container image in the first place. @@ -257,11 +283,10 @@ jobs: --device=/dev/kfd \ --device=/dev/dri \ --security-opt seccomp=unconfined \ - --pull always \ -v ${GITHUB_WORKSPACE}:/mnt/ais:ro \ -v ${{ env.AIS_MOUNT_PATH }}:/mnt/ais-fs \ --name ${AIS_CONTAINER_NAME} \ - ${AIS_CI_DEV_IMAGE} + ${{ env.AIS_CI_IMAGE_NAME }}_dev - name: Make copy of the code repository and create build directories run: | docker exec \ @@ -341,6 +366,13 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Download AIS CI image artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.AIS_CI_IMAGE_NAME }}_dev.tar + path: ${{ github.workspace }} + - name: Load AIS CI image into Docker + run: docker load --input ${GITHUB_WORKSPACE}/${{ env.AIS_CI_IMAGE_NAME }}_dev.tar # Detach the container and run separate commands to it. # Thus we can make separate explicit steps in the Github CI # as if we were able to parameterize the container image in the first place. @@ -349,11 +381,10 @@ jobs: docker run \ -dt \ --rm \ - --pull always \ -v ${GITHUB_WORKSPACE}:/mnt/ais:ro \ -v ${{ env.AIS_MOUNT_PATH }}:/mnt/ais-fs \ --name ${AIS_CONTAINER_NAME} \ - ${AIS_CI_DEV_IMAGE} + ${{ env.AIS_CI_IMAGE_NAME }}_dev - name: Make copy of the code repository and create build directories run: | docker exec \ diff --git a/.github/workflows/hipfile-nvidia.yml b/.github/workflows/hipfile-nvidia.yml index 2234e80e..9d4ef7bb 100644 --- a/.github/workflows/hipfile-nvidia.yml +++ b/.github/workflows/hipfile-nvidia.yml @@ -32,6 +32,13 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Download AIS CI image artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.AIS_CI_IMAGE_NAME }}_dev.tar + path: ${{ github.workspace }} + - name: Load AIS CI image into Docker + run: docker load --input ${GITHUB_WORKSPACE}/${{ env.AIS_CI_IMAGE_NAME }}_dev.tar # Detach the container and run separate commands to it. # Thus we can make separate explicit steps in the Github CI # as if we were able to parameterize the container image in the first place. @@ -44,12 +51,11 @@ jobs: -e NVIDIA_GDS=enabled \ --runtime=nvidia \ --gpus all \ - --pull always \ --cap-add=CAP_SYSLOG \ -v $(pwd):/mnt/ais:ro \ -v ${{ env.AIS_MOUNT_PATH }}:/mnt/ais-fs \ --name ${AIS_CONTAINER_NAME} \ - ${AIS_CI_DEV_IMAGE} + ${{ env.AIS_CI_IMAGE_NAME }}_dev - name: Make copy of the code repository run: | docker exec \ From d0a947452fb5e7c7b1f9c0aa67228e88ee1f8307 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Wed, 5 Nov 2025 13:10:21 -0700 Subject: [PATCH 2/4] Add workflow to update AIS CI image & cache This also provides a trigger to manually initiate an image update. A user must have at least write access to manually start the workflow. --- .github/workflows/update_ais_ci.yml | 55 +++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/update_ais_ci.yml diff --git a/.github/workflows/update_ais_ci.yml b/.github/workflows/update_ais_ci.yml new file mode 100644 index 00000000..7132a8c9 --- /dev/null +++ b/.github/workflows/update_ais_ci.yml @@ -0,0 +1,55 @@ +name: update_AIS_CI +run-name: Update latest CI image & cache +on: + pull_request_target: # A.K.A. This is a privileged action. See `pull_request`. + types: + - closed # CAUTION: Includes un-merged PR's. + branches: + - develop + workflow_dispatch: + # Requires write access to trigger. + +permissions: + contents: read + packages: write +jobs: + update_AIS_CI_image: + env: + AIS_DOCKER_REGISTRY: ghcr.io/rocm/hipfile + AIS_CI_IMAGE_NAME: ais_ci_${{ matrix.supported_platforms }} + if: ${{ github.event.pull_request.merged == true }} + runs-on: [ubuntu-24.04] + container: docker:28.5 + strategy: + matrix: + supported_platforms: + - rocky + - suse + - ubuntu + steps: + - name: Fetching code repository... + uses: actions/checkout@v5 + - name: Authenticating to GitHub Container Registry. + uses: docker/login-action@v3.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Setup Docker Builder + uses: docker/setup-buildx-action@v3.11.1 + with: + name: ais-builder + driver: docker-container + cache-binary: false # True uses the GHA Cache Backend. + # We won't use the cache from the previous image. + # This lets us semi-regularly pull in package updates automatically. + - name: Build & Push latest image for AIS CI + run: | + docker buildx build \ + -f ${GITHUB_WORKSPACE}/util/docker/DOCKERFILE.${{ env.AIS_CI_IMAGE_NAME }} \ + --label "org.opencontainers.image.description= \ + Latest AIS CI Image for ${{ matrix.supported_platforms }}." \ + --cache-to=type=registry,ref="${{ env.AIS_DOCKER_REGISTRY }}/${{ env.AIS_CI_IMAGE_NAME }}:cache" \ + --push \ + -t ${{ env.AIS_CI_IMAGE_NAME }}:latest \ + ${GITHUB_WORKSPACE} From 335fe0dc5a9c34ecceba9c9507a5cbe16af231bf Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Wed, 5 Nov 2025 15:59:30 -0700 Subject: [PATCH 3/4] Add missing commit for running dispatch jobs. --- .github/workflows/update_ais_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_ais_ci.yml b/.github/workflows/update_ais_ci.yml index 7132a8c9..48ac3bf1 100644 --- a/.github/workflows/update_ais_ci.yml +++ b/.github/workflows/update_ais_ci.yml @@ -17,7 +17,7 @@ jobs: env: AIS_DOCKER_REGISTRY: ghcr.io/rocm/hipfile AIS_CI_IMAGE_NAME: ais_ci_${{ matrix.supported_platforms }} - if: ${{ github.event.pull_request.merged == true }} + if: ${{ github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' }} runs-on: [ubuntu-24.04] container: docker:28.5 strategy: From 41521847a7889f50a4f481b8cc0fcee4f5392948 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Wed, 5 Nov 2025 21:54:30 -0700 Subject: [PATCH 4/4] Use new versions of the artifact jobs --- .github/workflows/build_ais.yml | 10 +++++----- .github/workflows/hipfile-nvidia.yml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_ais.yml b/.github/workflows/build_ais.yml index 798e5d62..27914b65 100644 --- a/.github/workflows/build_ais.yml +++ b/.github/workflows/build_ais.yml @@ -52,7 +52,7 @@ jobs: -t ${{ env.AIS_CI_IMAGE_NAME }}_dev \ ${GITHUB_WORKSPACE} - name: Upload AIS CI image as an artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: ${{ env.AIS_CI_IMAGE_NAME }}_dev.tar path: ${{ github.workspace }}/${{ env.AIS_CI_IMAGE_NAME }}_dev.tar @@ -79,7 +79,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Download AIS CI image artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 with: name: ${{ env.AIS_CI_IMAGE_NAME }}_dev.tar path: ${{ github.workspace }} @@ -195,7 +195,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Download AIS CI image artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 with: name: ${{ env.AIS_CI_IMAGE_NAME }}_dev.tar path: ${{ github.workspace }} @@ -266,7 +266,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Download AIS CI image artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 with: name: ${{ env.AIS_CI_IMAGE_NAME }}_dev.tar path: ${{ github.workspace }} @@ -367,7 +367,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Download AIS CI image artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 with: name: ${{ env.AIS_CI_IMAGE_NAME }}_dev.tar path: ${{ github.workspace }} diff --git a/.github/workflows/hipfile-nvidia.yml b/.github/workflows/hipfile-nvidia.yml index 9d4ef7bb..365820a7 100644 --- a/.github/workflows/hipfile-nvidia.yml +++ b/.github/workflows/hipfile-nvidia.yml @@ -33,7 +33,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Download AIS CI image artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 with: name: ${{ env.AIS_CI_IMAGE_NAME }}_dev.tar path: ${{ github.workspace }}