test-ais-system.yml

name: ais-system-tests
run-name: Run AIS System Tests
env:
  AIS_GHA_CMD_FILES_DOCKER_DIR: /root/github
  AIS_INPUT_HIPFILE_PKG_DEV_FILENAME: ${{ inputs.ais_hipfile_pkg_dev_filename }}
  AIS_INPUT_HIPFILE_PKG_FILENAME: ${{ inputs.ais_hipfile_pkg_filename }}
  AIS_INPUT_CI_IMAGE: ${{ inputs.ci_image }}
  AIS_INPUT_PLATFORM: ${{ inputs.platform }}
  AIS_INPUT_ROCM_VERSION: ${{ inputs.rocm_version }}
  AIS_MOUNT_PATH: /mnt/ais/ext4
  AIS_PKG_INSTALL_CMD: >-
    ${{
      case(
        inputs.platform == 'rocky', 'dnf install -y',
        inputs.platform == 'suse', 'zypper install -y --allow-unsigned-rpm',
        inputs.platform == 'ubuntu', 'apt update; apt install -y',
        ''
      )
    }}
  AIS_PKG_MGR: >-
    ${{
      inputs.platform == 'rocky' && 'dnf' ||
      inputs.platform == 'suse' && 'zypper' ||
      inputs.platform == 'ubuntu' && 'apt'
    }}
on:
  workflow_call:
    inputs:
      ais_hipfile_pkg_dev_filename:
        required: true
        type: string
      ais_hipfile_pkg_filename:
        required: true
        type: string
      ci_image:
        required: true
        type: string
      platform:
        required: true
        type: string
      rocm_version:
        description: "Target ROCm MAJOR.MINOR.PATCH Version."
        required: true
        type: string
permissions:
  contents: read
  packages: read
jobs:
  run_system_tests:
    runs-on: [linux, AIS]
    steps:
      - name: Set early AIS CI environment variables
        run: echo "AIS_PR_NUMBER=$(echo "${GITHUB_REF}" | sed 's|[^0-9]||g')" >> "${GITHUB_ENV}"
      - name: Set AIS CI container name
        run: |
          echo "AIS_CONTAINER_NAME=${AIS_PR_NUMBER}_${GITHUB_JOB}_${AIS_INPUT_PLATFORM}_${AIS_INPUT_ROCM_VERSION}" >> "${GITHUB_ENV}"
      - name: Fetching hipFile repository...
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
        with:
          path: hipFile
      - name: Download hipFile runtime package
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #v8.0.1
        with:
          name: ${{ inputs.ais_hipfile_pkg_filename }}
      - name: Download hipFile development package
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #v8.0.1
        with:
          name: ${{ inputs.ais_hipfile_pkg_dev_filename }}
      - name: Download hipFile build directory
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #v8.0.1
        with:
          name: hipfile-build-dir-${{ inputs.platform }}
          path: ${{ github.workspace }}/hipfile-build-dir
      - name: Authenticating to GitHub Container Registry
        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 #v4.0.0
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      # Detach the container and run separate commands to it.
      # Thus we can make separate explicit steps in the Github CI
      # as if we were able to parameterize the container image in the first place.
      - name: Starting Docker Container
        run: |
          docker run \
            -dt \
            --rm \
            --device=/dev/kfd \
            --device=/dev/dri \
            --security-opt seccomp=unconfined \
            --pull always \
            -v ${GITHUB_WORKSPACE}:/mnt/ais:ro \
            -v "${AIS_MOUNT_PATH}:/mnt/ais-fs" \
            --name "${AIS_CONTAINER_NAME}" \
            "${AIS_INPUT_CI_IMAGE}"
      - name: Create hipfile IO test directory
        run: |
          docker exec -t "${AIS_CONTAINER_NAME}" /bin/bash -c "mkdir -p /mnt/ais-fs/${AIS_CONTAINER_NAME}"
      - name: Make copy of the code repository and create build directories
        run: |
          docker exec \
            -t \
            "${AIS_CONTAINER_NAME}" \
            /bin/bash -c '
              cp -R /mnt/ais /ais
              mkdir /ais/hipFile/build
            '
      - name: Copy the hipFile packages into the container
        run: |
          docker cp \
            "${GITHUB_WORKSPACE}/${AIS_INPUT_HIPFILE_PKG_FILENAME}" \
            "${AIS_CONTAINER_NAME}:/root"
          docker cp \
            "${GITHUB_WORKSPACE}/${AIS_INPUT_HIPFILE_PKG_DEV_FILENAME}" \
            "${AIS_CONTAINER_NAME}:/root"
      - name: Install the hipFile packages
        run: |
          docker exec \
            -t \
            -w /root \
            "${AIS_CONTAINER_NAME}" \
            /bin/bash -c '
              ${{
                format(
                  env.AIS_PKG_MGR == 'apt' && 'apt install -y "./{0}" "./{1}"' ||
                    env.AIS_PKG_MGR == 'dnf' && 'dnf install -y --cacheonly "./{0}" "./{1}"' ||
                    env.AIS_PKG_MGR == 'zypper' && 'zypper --no-refresh install -y --allow-unsigned-rpm "./{0}" "./{1}"' ||
                    'echo "Unknown platform."; exit 1',
                  inputs.ais_hipfile_pkg_filename,
                  inputs.ais_hipfile_pkg_dev_filename
                )
              }}
            '
      # GitHub Build Artifacts do not preserve file permissions, namely the executable bit.
      - name: Fix file permissions on GTest executables
        run: |
          chmod a+x \
            ${GITHUB_WORKSPACE}/hipfile-build-dir/test/hipfile_system_tests \
            ${GITHUB_WORKSPACE}/hipfile-build-dir/test/amd_detail/batch_mt \
            ${GITHUB_WORKSPACE}/hipfile-build-dir/test/amd_detail/state_mt \
            ${GITHUB_WORKSPACE}/hipfile-build-dir/examples/aiscp/aiscp
      - name: Copy the hipFile build directory into the container
        run: |
          docker cp \
            ${GITHUB_WORKSPACE}/hipfile-build-dir/. \
            "${AIS_CONTAINER_NAME}:/ais/hipFile/build"
      - name: Run hipFile system tests for the AMD platform (amdclang++)
        run: |
          docker exec \
            -t \
            -w /ais/hipFile/build \
            "${AIS_CONTAINER_NAME}" \
            /bin/bash -c '
              ctest -V -L "system" --parallel
            '
      - name: Stress tests for hipFile targeting the AMD platform (amdclang++)
        run: |
          docker exec \
            -t \
            -w /ais/hipFile/build \
            "${AIS_CONTAINER_NAME}" \
            /bin/bash -c '
              ctest -V -L "stress" --parallel
            '
      - name: hipFile fallback/POSIX IO test using aiscp
        run: |
          docker exec \
            -t \
            -w "/mnt/ais-fs/${AIS_CONTAINER_NAME}" \
            "${AIS_CONTAINER_NAME}" \
            /bin/bash -c '
              HIPFILE_FORCE_COMPAT_MODE=true
                /ais/hipFile/util/ci-aiscp-test.sh \
                /ais/hipFile/build/examples/aiscp/aiscp
            '
      - name: hipFile fastpath/AIS IO test using aiscp
        run: |
          docker exec \
            -t \
            -w "/mnt/ais-fs/${AIS_CONTAINER_NAME}" \
            "${AIS_CONTAINER_NAME}" \
            /bin/bash -c '
              HIPFILE_ALLOW_COMPAT_MODE=false
                /ais/hipFile/util/ci-aiscp-test.sh \
                /ais/hipFile/build/examples/aiscp/aiscp
            '
      - name: Destroy hipfile IO test directory
        if: ${{ always() }}
        run: |
          docker exec -t "${AIS_CONTAINER_NAME}" /bin/bash -c "rm -fr /mnt/ais-fs/${AIS_CONTAINER_NAME}"
      - name: Cleanup & Stop the Docker container
        if: ${{ always() }}
        run: |
          docker stop "${AIS_CONTAINER_NAME}"
      - name: Cleanup self-hosted runner workspace
        if: ${{ always() }}
        run: |
          shopt -s dotglob
          rm -rf ${GITHUB_WORKSPACE}/*
  build_FIO:
    uses: ROCm/fio/.github/workflows/build-fio.yml@hipFile
    with:
      ais_hipfile_pkg_filename: ${{ inputs.ais_hipfile_pkg_filename }}
      ais_hipfile_pkg_dev_filename: ${{ inputs.ais_hipfile_pkg_dev_filename }}
      platform: ${{ inputs.platform }}
  run_FIO_tests:
    runs-on: [linux, AIS]
    needs: [build_FIO]
    env:
      FIO_RUNTIME_PKG_FILENAME: >-
        ${{
          case(
            inputs.platform == 'rocky', needs.build_FIO.outputs.fio_pkg_fedora_filename,
            inputs.platform == 'suse', needs.build_FIO.outputs.fio_pkg_suse_filename,
            inputs.platform == 'ubuntu', needs.build_FIO.outputs.fio_pkg_debian_filename,
            ''
          )
        }}
      FIO_HIPFILE_ENGINE_PKG_FILENAME: >-
        ${{
          case(
            inputs.platform == 'rocky', needs.build_FIO.outputs.fio_pkg_fedora_hipfile_engine_filename,
            ''
          )
        }}
    steps:
      - name: Set early AIS CI environment variables
        run: echo "AIS_PR_NUMBER=$(echo "${GITHUB_REF}" | sed 's|[^0-9]||g')" >> "${GITHUB_ENV}"
      - name: Set AIS CI container name
        run: |
          echo "AIS_CONTAINER_NAME=${AIS_PR_NUMBER}_${GITHUB_JOB}_${AIS_INPUT_PLATFORM}_${AIS_INPUT_ROCM_VERSION}" >> "${GITHUB_ENV}"
      # hipFile repo needed for //util directory
      - name: Fetching hipFile repository...
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
        with:
          path: hipFile
      - name: Download hipFile runtime package
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #v8.0.1
        with:
          name: ${{ inputs.ais_hipfile_pkg_filename }}
      - name: Download hipFile development package
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #v8.0.1
        with:
          name: ${{ inputs.ais_hipfile_pkg_dev_filename }}
      - name: Download FIO runtime package
        # Fedora-based distro's need to download multiple packages as the FIO IO engines
        # are dynamically linked instead of built into the FIO executable.
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #v8.0.1
        with:
          artifact-ids: >-
            ${{
              case(
                inputs.platform == 'rocky',
                format(
                  '{0},{1}',
                  needs.build_FIO.outputs.fio_pkg_fedora_artifact_id,
                  needs.build_FIO.outputs.fio_pkg_fedora_hipfile_engine_artifact_id
                ),
                inputs.platform == 'suse', needs.build_FIO.outputs.fio_pkg_suse_artifact_id,
                inputs.platform == 'ubuntu', needs.build_FIO.outputs.fio_pkg_debian_artifact_id,
                format('Platform "{0}" is not supported by build_FIO.', inputs.platform)
              )
            }}
          merge-multiple: true
      - name: Authenticating to GitHub Container Registry
        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 #v4.0.0
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Starting Docker Container
        run: |
          docker run \
            -dt \
            --rm \
            --device=/dev/kfd \
            --device=/dev/dri \
            --security-opt seccomp=unconfined \
            --pull always \
            -v ${GITHUB_WORKSPACE}:/mnt/ais:ro \
            -v "${AIS_MOUNT_PATH}:/mnt/ais-fs" \
            --name "${AIS_CONTAINER_NAME}" \
            "${AIS_INPUT_CI_IMAGE}"
      - name: Create hipfile IO test directory
        run: |
          docker exec -t "${AIS_CONTAINER_NAME}" /bin/bash -c "mkdir -p /mnt/ais-fs/${AIS_CONTAINER_NAME}"
      - name: Copy the hipFile packages into the container
        run: |
          docker cp \
            "${GITHUB_WORKSPACE}/${AIS_INPUT_HIPFILE_PKG_FILENAME}" \
            "${AIS_CONTAINER_NAME}:/root"
          docker cp \
            "${GITHUB_WORKSPACE}/${AIS_INPUT_HIPFILE_PKG_DEV_FILENAME}" \
            "${AIS_CONTAINER_NAME}:/root"
      - name: Install the hipFile packages
        # Note: We need to re-run the `ldconfig` post-install step.
        #       This really should just be done automatically by the ROCm package.
        run: |
          docker exec \
            -t \
            -w /root \
            "${AIS_CONTAINER_NAME}" \
            /bin/bash -c '
              ${{
                format(
                  env.AIS_PKG_MGR == 'apt' && 'apt update; apt install -y "./{0}" "./{1}"' ||
                    env.AIS_PKG_MGR == 'dnf' && 'dnf install -y --cacheonly "./{0}" "./{1}"' ||
                    env.AIS_PKG_MGR == 'zypper' && 'zypper --no-refresh install -y --allow-unsigned-rpm "./{0}" "./{1}"' ||
                    'echo "Unknown platform."; exit 1',
                  inputs.ais_hipfile_pkg_filename,
                  inputs.ais_hipfile_pkg_dev_filename
                )
              }}
              ldconfig
            '
      - name: Copy the FIO packages into the container
        run: |
          docker cp \
            "${GITHUB_WORKSPACE}/${FIO_RUNTIME_PKG_FILENAME}" \
            "${AIS_CONTAINER_NAME}:/root"
          ${{
            case(
              inputs.platform == 'rocky',
                'docker cp \
                  "${GITHUB_WORKSPACE}/${FIO_HIPFILE_ENGINE_PKG_FILENAME}" \
                  "${AIS_CONTAINER_NAME}:/root"
                ',
              ''
            )
          }}
      - name: Install the FIO packages
        run: |
          docker exec \
            -t \
            -w /root \
            "${AIS_CONTAINER_NAME}" \
            /bin/bash -c '
              ${{
                format(
                  '{0} {1}',
                  env.AIS_PKG_INSTALL_CMD,
                  case(
                    inputs.platform == 'rocky',
                      format('./{0} ./{1}', env.FIO_RUNTIME_PKG_FILENAME, env.FIO_HIPFILE_ENGINE_PKG_FILENAME),
                    format('./{0}', env.FIO_RUNTIME_PKG_FILENAME)
                  )
                )
              }}
            '
      - name: hipFile fallback/POSIX IO test using fio
        run: |
          docker exec \
            -t \
            -w /mnt/ais-fs/${AIS_CONTAINER_NAME} \
            "${AIS_CONTAINER_NAME}" \
            /bin/bash -c '
              HIPFILE_FORCE_COMPAT_MODE=true \
                fio \
                /mnt/ais/hipFile/util/fio/write-read-verify.fio
            '
      - name: hipFile AIS IO test using fio
        run: |
          docker exec \
            -t \
            -w /mnt/ais-fs/${AIS_CONTAINER_NAME} \
            "${AIS_CONTAINER_NAME}" \
            /bin/bash -c '
              HIPFILE_ALLOW_COMPAT_MODE=false \
                fio \
                /mnt/ais/hipFile/util/fio/write-read-verify.fio
            '
      - name: Destroy hipfile IO test directory
        if: ${{ always() }}
        run: |
          docker exec -t "${AIS_CONTAINER_NAME}" /bin/bash -c "rm -fr /mnt/ais-fs/${AIS_CONTAINER_NAME}"
      - name: Cleanup & Stop the Docker container
        if: ${{ always() }}
        run: |
          docker stop "${AIS_CONTAINER_NAME}"
      - name: Cleanup self-hosted runner workspace
        if: ${{ always() }}
        run: |
          shopt -s dotglob
          rm -rf ${GITHUB_WORKSPACE}/*