diff --git a/.github/workflows/ais-ci.yml b/.github/workflows/ais-ci.yml new file mode 100644 index 00000000..f58a5186 --- /dev/null +++ b/.github/workflows/ais-ci.yml @@ -0,0 +1,59 @@ +name: AIS_CI +run-name: Main CI workflow to coordinate jobs for a single platform. +on: + workflow_call: + inputs: + platform: + required: true + type: string +permissions: + contents: read + packages: read +jobs: + AIS_CI_Pre-check: + outputs: + changed_dockerfile: ${{ steps.ci-flags.outputs.changed_dockerfile }} + runs-on: [ubuntu-24.04] + steps: + - name: Fetching code repository... + uses: actions/checkout@v5 + - name: Git fetch base ref + run: git fetch origin ${{ github.base_ref }} + - name: Set CI Flags + id: ci-flags + run: | + echo "changed_dockerfile=$(./util/files-changed.sh origin/${{ github.base_ref }} HEAD 'util/docker/DOCKERFILE.*')" >> ${GITHUB_OUTPUT} + build_AIS_CI_image: + if: ${{ needs.AIS_CI_Pre-check.outputs.changed_dockerfile == '1' }} + needs: AIS_CI_Pre-check + permissions: + contents: read + packages: write + uses: ./.github/workflows/build-ais-ci-image.yml + with: + platform: ${{ inputs.platform }} + build_and_test: + # Run after build_AIS_CI_image but only if build_AIS_CI_image passed or was skipped. + # always() check needed otherwise success() implicitly added. + if: >- + ${{ + always() && + ( + needs.build_AIS_CI_image.result == 'skipped' || + ( + needs['AIS_CI_Pre-check'].outputs.changed_dockerfile == '1' && + needs.build_AIS_CI_image.result == 'success' + ) + ) + }} + needs: [AIS_CI_Pre-check, build_AIS_CI_image] + uses: ./.github/workflows/build-ais.yml + with: + # If Dockerfile changed, use the new CI image. Otherwise, use 'latest'. + ci_image: >- + ${{ + needs.AIS_CI_Pre-check.outputs.changed_dockerfile == '1' && + needs.build_AIS_CI_image.outputs.ci_image || + format('ghcr.io/rocm/hipfile/ais_ci_{0}:latest', inputs.platform) + }} + platform: ${{ inputs.platform }} diff --git a/.github/workflows/build-ais-ci-image.yml b/.github/workflows/build-ais-ci-image.yml new file mode 100644 index 00000000..08a84203 --- /dev/null +++ b/.github/workflows/build-ais-ci-image.yml @@ -0,0 +1,67 @@ +name: new_CI_image +run-name: Build the Docker image for AIS CI. +env: + AIS_DOCKER_REGISTRY: ghcr.io/rocm/hipfile + AIS_PR_BASE_URL: https://github.com/ROCm/hipFile/pull +on: + workflow_call: + inputs: + platform: + required: true + type: string + outputs: + ci_image: + description: The full name & tag of the newly built image. + value: ${{ jobs.build_AIS_image.outputs.new_ci_image}} +permissions: + contents: read + packages: write +jobs: + build_AIS_image: + env: + AIS_CI_IMAGE_NAME: ais_ci_${{ inputs.platform }} + runs-on: [ubuntu-24.04] + container: docker:28.5 + outputs: + new_ci_image: ${{ steps.ci-image.outputs.AIS_CI_IMAGE }} + image_tag: ${{ steps.ci-image.outputs.AIS_CI_IMAGE_TAG }} + # image_tag is a bit hacky. GitHub does not support separating outputs + # from a workflow run in a matrix. The last workflow to run will set the + # output value. `image_tag` is constant across matrix jobs so this is fine. + steps: + - name: Set CI environment variables + run: | + echo "AIS_CI_IMAGE_TAG=$(echo ${{ github.ref }} | sed 's|[^a-zA-Z0-9]|-|g')" >> "${GITHUB_ENV}" + echo "AIS_PR_NUMBER=$(echo ${{ github.ref }} | sed 's|[^0-9]||g')" >> "${GITHUB_ENV}" + - name: Set target AIS CI Image + id: ci-image + run: | + echo "AIS_CI_IMAGE=${{ env.AIS_DOCKER_REGISTRY }}/${{ env.AIS_CI_IMAGE_NAME}}:${AIS_CI_IMAGE_TAG}" \ + >> "${GITHUB_OUTPUT}" + - name: Fetching code repository... + uses: actions/checkout@v5 + - name: Authenticating to GitHub Container Registry. + uses: docker/login-action@v3.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Setup Docker Builder + uses: docker/setup-buildx-action@v3.11.1 + with: + name: ais-builder + driver: docker-container + cache-binary: false # True uses the GHA Cache Backend. + - name: Build base image for AIS CI + run: | + docker buildx build \ + -f ${GITHUB_WORKSPACE}/util/docker/DOCKERFILE.${{ env.AIS_CI_IMAGE_NAME }} \ + --label "org.opencontainers.image.description= \ + ${{ env.AIS_CI_IMAGE_NAME }} Development Image for AIS CI using branch \ + ${{ github.head_ref }} for PR #${AIS_PR_NUMBER}. \ + PR URL: ${{ env.AIS_PR_BASE_URL }}/${AIS_PR_NUMBER}" \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --cache-from=type=registry,ref=${{ steps.ci-image.outputs.AIS_CI_IMAGE }} \ + --push \ + -t ${{ steps.ci-image.outputs.AIS_CI_IMAGE }} \ + ${GITHUB_WORKSPACE} diff --git a/.github/workflows/build_ais.yml b/.github/workflows/build-ais.yml similarity index 80% rename from .github/workflows/build_ais.yml rename to .github/workflows/build-ais.yml index 9362e579..36cb95e5 100644 --- a/.github/workflows/build_ais.yml +++ b/.github/workflows/build-ais.yml @@ -1,67 +1,27 @@ -name: AIS # Mono-workflow - May be advantageous to split up +name: build_and_test # Mono-workflow - May be advantageous to split up run-name: Build, Test, and Analyze AIS env: AIS_MOUNT_PATH: /mnt/ais/ext4 - AIS_DOCKER_REGISTRY: ghcr.io/rocm/hipfile - AIS_CI_IMAGE_NAME: ais_ci_${{inputs.platform}} - AIS_PR_BASE_URL: https://github.com/ROCm/hipFile/pull on: workflow_call: inputs: + ci_image: + required: true + type: string platform: required: true type: string permissions: contents: read - packages: write + packages: read jobs: - build_AIS_image: - runs-on: [ubuntu-24.04] - container: docker:28.5 - steps: - - name: Set AIS CI image environment variables - run: | - echo "AIS_CI_DEV_IMAGE=${{ env.AIS_DOCKER_REGISTRY }}/${{ env.AIS_CI_IMAGE_NAME }}_dev:$(echo ${{ github.ref }} \ - | sed 's|[^a-zA-Z0-9]|-|g')" >> "$GITHUB_ENV" - echo "AIS_CI_LATEST_IMAGE=${{ env.AIS_DOCKER_REGISTRY }}/${{ env.AIS_CI_IMAGE_NAME }}:latest" >> "$GITHUB_ENV" - echo "AIS_PR_NUMBER=$(echo ${{ github.ref }} | sed 's|[^0-9]||g')" >> "$GITHUB_ENV" - - name: Fetching code repository... - uses: actions/checkout@v5 - - name: Authenticating to GitHub Container Registry. - uses: docker/login-action@v3.6.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Setup Docker Builder - uses: docker/setup-buildx-action@v3.11.1 - with: - name: ais-builder - driver: docker-container - cache-binary: false # True uses the GHA Cache Backend. - - name: Build base image for AIS CI - run: | - docker buildx build \ - -f ${GITHUB_WORKSPACE}/util/docker/DOCKERFILE.${{ env.AIS_CI_IMAGE_NAME }} \ - --label "org.opencontainers.image.description= \ - ${{ env.AIS_CI_IMAGE_NAME }} Development Image for AIS CI using branch \ - ${{ github.head_ref }} for PR #${AIS_PR_NUMBER}. \ - PR URL: ${{ env.AIS_PR_BASE_URL }}/${AIS_PR_NUMBER}" \ - --cache-to=type=registry,ref="${{ env.AIS_DOCKER_REGISTRY }}/${{ env.AIS_CI_IMAGE_NAME }}_dev:cache" \ - --cache-from=type=registry,ref="${{ env.AIS_DOCKER_REGISTRY }}/${{ env.AIS_CI_IMAGE_NAME }}_dev:cache" \ - --push \ - -t ${AIS_CI_DEV_IMAGE} \ - ${GITHUB_WORKSPACE} compile_on_AMD: runs-on: [ubuntu-24.04] - needs: build_AIS_image steps: - name: Get PR number and store it as a environment variable run: echo "AIS_PR_NUMBER=$(echo ${{ github.ref }} | sed 's|[^0-9]||g')" >> "$GITHUB_ENV" - name: Set AIS CI image environment variables run: | - echo "AIS_CI_DEV_IMAGE=${{ env.AIS_DOCKER_REGISTRY }}/${{ env.AIS_CI_IMAGE_NAME }}_dev:$(echo ${{ github.ref }} \ - | sed 's|[^a-zA-Z0-9]|-|g')" >> "$GITHUB_ENV" echo "AIS_CONTAINER_NAME=${AIS_PR_NUMBER}_${{ github.job }}_${{inputs.platform}}" >> "$GITHUB_ENV" - name: Fetching code repository... uses: actions/checkout@v5 @@ -83,7 +43,7 @@ jobs: -v ${GITHUB_WORKSPACE}:/mnt/ais:ro \ -v ${{ env.AIS_MOUNT_PATH }}:/mnt/ais-fs \ --name ${AIS_CONTAINER_NAME} \ - ${AIS_CI_DEV_IMAGE} + ${{ inputs.ci_image }} - name: Make copy of the code repository and create build directories # Single quotes necessary to ensure string/command substitutions happen # in the container and not on the host. @@ -158,7 +118,6 @@ jobs: docker stop ${AIS_CONTAINER_NAME} compile_on_AMD_other: runs-on: [ubuntu-24.04] - needs: build_AIS_image strategy: fail-fast: false matrix: @@ -170,8 +129,6 @@ jobs: run: echo "AIS_PR_NUMBER=$(echo ${{ github.ref }} | sed 's|[^0-9]||g')" >> "$GITHUB_ENV" - name: Set AIS CI image environment variables run: | - echo "AIS_CI_DEV_IMAGE=${{ env.AIS_DOCKER_REGISTRY }}/${{ env.AIS_CI_IMAGE_NAME }}_dev:$(echo ${{ github.ref }} \ - | sed 's|[^a-zA-Z0-9]|-|g')" >> "$GITHUB_ENV" echo "AIS_CONTAINER_NAME=${AIS_PR_NUMBER}_${{ github.job }}_${{inputs.platform}}" >> "$GITHUB_ENV" - name: Fetching code repository... uses: actions/checkout@v5 @@ -190,7 +147,7 @@ jobs: -v ${GITHUB_WORKSPACE}:/mnt/ais:ro \ -v ${{ env.AIS_MOUNT_PATH }}:/mnt/ais-fs \ --name ${AIS_CONTAINER_NAME} \ - ${AIS_CI_DEV_IMAGE} + ${{ inputs.ci_image }} - name: Make copy of the code repository and create build directories run: | docker exec \ @@ -235,8 +192,6 @@ jobs: run: echo "AIS_PR_NUMBER=$(echo ${{ github.ref }} | sed 's|[^0-9]||g')" >> "$GITHUB_ENV" - name: Set AIS CI image environment variables run: | - echo "AIS_CI_DEV_IMAGE=${{ env.AIS_DOCKER_REGISTRY }}/${{ env.AIS_CI_IMAGE_NAME }}_dev:$(echo ${{ github.ref }} \ - | sed 's|[^a-zA-Z0-9]|-|g')" >> "$GITHUB_ENV" echo "AIS_CONTAINER_NAME=${AIS_PR_NUMBER}_${{ github.job }}_${{inputs.platform}}_nvidia" >> "$GITHUB_ENV" - name: Fetching code repository... uses: actions/checkout@v5 @@ -261,7 +216,7 @@ jobs: -v ${GITHUB_WORKSPACE}:/mnt/ais:ro \ -v ${{ env.AIS_MOUNT_PATH }}:/mnt/ais-fs \ --name ${AIS_CONTAINER_NAME} \ - ${AIS_CI_DEV_IMAGE} + ${{ inputs.ci_image }} - name: Make copy of the code repository and create build directories run: | docker exec \ @@ -318,7 +273,6 @@ jobs: docker stop ${AIS_CONTAINER_NAME} compile_on_NVIDIA: runs-on: [ubuntu-24.04] - needs: build_AIS_image strategy: fail-fast: false matrix: @@ -330,8 +284,6 @@ jobs: run: echo "AIS_PR_NUMBER=$(echo ${{ github.ref }} | sed 's|[^0-9]||g')" >> "$GITHUB_ENV" - name: Set AIS CI image environment variables run: | - echo "AIS_CI_DEV_IMAGE=${{ env.AIS_DOCKER_REGISTRY }}/${{ env.AIS_CI_IMAGE_NAME }}_dev:$(echo ${{ github.ref }} \ - | sed 's|[^a-zA-Z0-9]|-|g')" >> "$GITHUB_ENV" echo "AIS_CONTAINER_NAME=${AIS_PR_NUMBER}_${{ github.job }}_${{inputs.platform}}_nvidia" >> "$GITHUB_ENV" - name: Fetching code repository... uses: actions/checkout@v5 @@ -353,7 +305,7 @@ jobs: -v ${GITHUB_WORKSPACE}:/mnt/ais:ro \ -v ${{ env.AIS_MOUNT_PATH }}:/mnt/ais-fs \ --name ${AIS_CONTAINER_NAME} \ - ${AIS_CI_DEV_IMAGE} + ${{ inputs.ci_image }} - name: Make copy of the code repository and create build directories run: | docker exec \ diff --git a/.github/workflows/ais_ci.yml b/.github/workflows/root-ci.yml similarity index 50% rename from .github/workflows/ais_ci.yml rename to .github/workflows/root-ci.yml index 77e854d4..202d77ce 100644 --- a/.github/workflows/ais_ci.yml +++ b/.github/workflows/root-ci.yml @@ -1,5 +1,12 @@ -name: AIS_CI +name: CI run-name: Kicks off several AIS CI workflows for various platforms. + +# In particular, this root workflow's only purpose is to allow +# us to put our entire CI in a matrix. It has the added benefit +# of not needing to specify our supported_platforms in multiple +# locations. +# We can consider moving the Pre-Checks here and passing them into +# the CI as an input. on: pull_request: types: [opened, synchronize, reopened] # Defaults @@ -9,15 +16,18 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: + contents: read + packages: write jobs: AIS_CI: strategy: fail-fast: false matrix: supported_platforms: - - ubuntu - rocky - suse - uses: ./.github/workflows/build_ais.yml + - ubuntu + uses: ./.github/workflows/ais-ci.yml with: - platform: ${{matrix.supported_platforms}} + platform: ${{ matrix.supported_platforms }} diff --git a/.github/workflows/update-ais-ci-image.yml b/.github/workflows/update-ais-ci-image.yml new file mode 100644 index 00000000..c75237a8 --- /dev/null +++ b/.github/workflows/update-ais-ci-image.yml @@ -0,0 +1,59 @@ +name: update_CI_image +run-name: Update latest CI image +on: + pull_request_target: # A.K.A. This is a privileged action. See `pull_request`. + types: + - closed # CAUTION: Includes un-merged PR's. + branches: + - develop + workflow_dispatch: + # Requires write access to trigger. + +permissions: + contents: read + packages: write +jobs: + update_AIS_CI_image: + env: + AIS_DOCKER_REGISTRY: ghcr.io/rocm/hipfile + AIS_CI_IMAGE_NAME: ais_ci_${{ matrix.supported_platforms }} + if: ${{ github.event.pull_request.merged == true }} + runs-on: [ubuntu-24.04] + container: docker:28.5 + strategy: + matrix: + supported_platforms: + - rocky + - suse + - ubuntu + steps: + - name: Fetching code repository... + uses: actions/checkout@v5 + - name: Authenticating to GitHub Container Registry. + uses: docker/login-action@v3.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set target AIS CI Image + id: ci-image + run: | + echo "AIS_CI_IMAGE=${{ env.AIS_DOCKER_REGISTRY }}/${{ env.AIS_CI_IMAGE_NAME}}:latest" \ + >> "${GITHUB_OUTPUT}" + - name: Setup Docker Builder + uses: docker/setup-buildx-action@v3.11.1 + with: + name: ais-builder + driver: docker-container + cache-binary: false # True uses the GHA Cache Backend. + # We won't use the cache from the previous image. + # This lets us semi-regularly pull in package updates automatically. + - name: Build & Push latest image for AIS CI + run: | + docker buildx build \ + -f ${GITHUB_WORKSPACE}/util/docker/DOCKERFILE.${{ env.AIS_CI_IMAGE_NAME }} \ + --label "org.opencontainers.image.description= \ + Latest AIS CI Image for ${{ matrix.supported_platforms }}." \ + --push \ + -t ${{ steps.ci-image.outputs.AIS_CI_IMAGE }} \ + ${GITHUB_WORKSPACE} \ No newline at end of file diff --git a/util/files-changed.sh b/util/files-changed.sh new file mode 100755 index 00000000..5115fac1 --- /dev/null +++ b/util/files-changed.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +# +# SPDX-License-Identifier: MIT + +# CI Script +# ./files-changed.sh BASE_REF HEAD_REF PATTERN +# Prints 0 if no matching files have changed +# Prints 1 if at least 1 matched file has changed. + +set -eo pipefail + +BASE_REF=$1 +HEAD_REF=$2 +CHECK_PATTERN=$3 +FILES_CHANGED=0 + +changed_files=() +if ! diff_output="$(git diff --name-only ${BASE_REF} ${HEAD_REF})"; then + echo ${diff_output} + exit 1 +fi +mapfile -t changed_files <<< "${diff_output}" + +for changed_file in "${changed_files[@]}"; do + if [[ "${changed_file}" == ${CHECK_PATTERN} ]]; then + FILES_CHANGED=1 + fi +done + +printf '%s' ${FILES_CHANGED} +exit 0 \ No newline at end of file