Build and deploy python wheels to GitHub Releases #4
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and deploy python wheels to GitHub Releases | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| debug_enabled: | |
| type: boolean | |
| description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' | |
| required: false | |
| default: false | |
| push: | |
| tags: | |
| - "v*.*.*" | |
| jobs: | |
| build_manylinux_wheels: | |
| name: Build linux wheels with ${{ matrix.py_ver_prefix }}-${{ matrix.linux_arch }}-${{ matrix.device }} | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| py_ver_prefix: [cp36, cp37, cp38, cp39, cp310, cp311, cp312] | |
| linux_arch: [x86_64, aarch64, s390x, ppc64le] #the maximum timeout of a job is 360 minutes, which is easy to time out, so we compile different system architectures separately. | |
| device: [cpu, gpu] | |
| exclude: | |
| - linux_arch: aarch64 | |
| device: gpu | |
| - linux_arch: s390x | |
| device: gpu | |
| - linux_arch: ppc64le | |
| device: gpu | |
| fail-fast: false | |
| timeout-minutes: 1440 | |
| steps: | |
| - name: Check out source code | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| - name: Remove unnecessary directories to free up space | |
| run: | | |
| sudo rm -rf /usr/local/.ghcup | |
| sudo rm -rf /opt/hostedtoolcache/CodeQL | |
| sudo rm -rf /usr/local/lib/android/sdk/ndk | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /usr/local/share/boost | |
| - name: Get python path | |
| run: | | |
| sudo apt update | |
| sudo apt install -y jq | |
| echo PY_PATH=$(echo '{"cp36": "cp36-cp36m", "cp37": "cp37-cp37m", "cp38": "cp38-cp38", "cp39": "cp39-cp39", "cp310": "cp310-cp310", "cp311": "cp311-cp311", "cp312": "cp312-cp312", "pp37": "pp37-pypy37_pp73", "pp38": "pp38-pypy38_pp73", "pp39": "pp39-pypy39_pp73"}' | jq .${{matrix.py_ver_prefix}} ) >> $GITHUB_ENV | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - name: Get necessary envs that the current process has access to | |
| id: set_env | |
| run: | | |
| from os import environ, sched_getaffinity | |
| # get the cpu cores | |
| num_cpus = len(sched_getaffinity(0)) | |
| output_file = environ["GITHUB_OUTPUT"] | |
| with open(output_file, "a", encoding="utf-8") as output_stream: | |
| output_stream.write(f"count={num_cpus}\n") | |
| shell: python | |
| - name: Build wheels | |
| uses: pypa/cibuildwheel@v2.21.3 | |
| env: | |
| CIBW_ENVIRONMENT: > | |
| LD_LIBRARY_PATH=/opt/python/${{ env.PY_PATH }}/lib:/opt/python/cp38-cp38/lib:/opt/python/cp310-cp310/lib | |
| CMAKE_BUILD_PARALLEL_LEVEL=${{ steps.set_env.outputs.count }} | |
| GIT_SHA=${{ github.sha }} | |
| DEVICE=${{ matrix.device }} | |
| CUDA_PATH=/usr/local/cuda | |
| SCRIPT_EXEC_MODE=x86 | |
| CIBW_BUILD_VERBOSITY: 1 | |
| CIBW_BUILD: ${{ matrix.py_ver_prefix }}-* | |
| CIBW_SKIP: "*musllinux*" #disable musllinux now | |
| CIBW_ARCHS_LINUX: ${{ matrix.linux_arch }} | |
| #CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/${{ github.repository }}_manylinux2014_x86_64_shared_python:latest | |
| #CIBW_MANYLINUX_I686_IMAGE: ghcr.io/${{ github.repository }}_manylinux2014_i686_shared_python:latest | |
| #CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/${{ github.repository }}_manylinux2014_aarch64_shared_python:latest | |
| #CIBW_MANYLINUX_PPC64LE_IMAGE: ghcr.io/${{ github.repository }}_manylinux2014_ppc64le_shared_python:latest | |
| #CIBW_MANYLINUX_S390X_IMAGE: ghcr.io/${{ github.repository }}_manylinux2014_s390x_shared_python:latest | |
| CIBW_MANYLINUX_X86_64_IMAGE: babitmf/manylinux_2_28_x86_64_shared_python:latest | |
| CIBW_MANYLINUX_AARCH64_IMAGE: babitmf/manylinux_2_28_aarch64_shared_python:latest | |
| CIBW_MANYLINUX_PPC64LE_IMAGE: babitmf/manylinux_2_28_ppc64le_shared_python:latest | |
| CIBW_MANYLINUX_S390X_IMAGE: babitmf/manylinux_2_28_s390x_shared_python:latest | |
| CIBW_BEFORE_ALL_LINUX: > | |
| (./scripts/build_ffmpeg.sh --device=${{ matrix.device }}) | |
| CIBW_REPAIR_WHEEL_COMMAND_LINUX: | |
| tmp_dir=$(mktemp -d) && | |
| unzip -d ${tmp_dir} {wheel} && | |
| auditwheel -v repair $(ldd ${tmp_dir}/bmf/lib/* | awk '{if(NF>1){print $1}}' | grep "avdevice\|avfilter\|avformat\|avcodec\|postproc\|swresample\|swscale\|avutil\|cuda\|libnpp\|libxcb\|libXau\|libbz2\|bmf_module_sdk\|engine\|hmp" | sort | uniq | awk '{printf(" --exclude %s", $1);}') -w {dest_dir} {wheel} && | |
| rm -rf ${tmp_dir} | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: Build linux wheels with ${{ matrix.py_ver_prefix }}-${{ matrix.linux_arch }}-${{ matrix.device }} | |
| path: ./wheelhouse/*.whl | |
| # build_macos_wheels: | |
| # name: Build macos wheels with ${{ matrix.py_ver_prefix }}-${{ matrix.macos_arch }} | |
| # runs-on: macos-14 # On the macos-14 system, the wheel package generated by using the xcode14.0~14.3 version compiler will have problems at python runtime which installed by homebrew, and the error is reported: "Symbol not found: _PyBaseObject_Type", please refer to issue: https://github.com/pybind/pybind11/pull/4301 and https://github.com/python/cpython/issues/97524. So we use macos-11 here. | |
| # strategy: | |
| # matrix: | |
| # py_ver_prefix: [cp36, cp37, cp38, cp39, cp310, cp311, cp312] | |
| # macos_arch: [x86_64, arm64, universal2] | |
| # exclude: | |
| # - py_ver_prefix: cp36 | |
| # macos_arch: arm64 | |
| # - py_ver_prefix: cp36 | |
| # macos_arch: universal2 | |
| # - py_ver_prefix: cp37 | |
| # macos_arch: arm64 | |
| # - py_ver_prefix: cp37 | |
| # macos_arch: universal2 | |
| # - py_ver_prefix: cp38 # the universal python package is experimental on Intel host, so we disable cross-compiling for python3.8. | |
| # macos_arch: arm64 | |
| # - py_ver_prefix: cp38 | |
| # macos_arch: universal2 | |
| # fail-fast: false | |
| # timeout-minutes: 1440 | |
| # steps: | |
| # - name: Check out source code | |
| # uses: actions/checkout@v3 | |
| # with: | |
| # submodules: 'recursive' | |
| # - name: Build wheels | |
| # uses: pypa/cibuildwheel@v2.21.3 | |
| # env: | |
| # #TODO get real cpu counts | |
| # # HAVE_STD_REGEX and RUN_HAVE_STD_REGES is needed by google benchmark when compiling ARM64 on x86_64 for macos. | |
| # CIBW_ENVIRONMENT: CMAKE_BUILD_PARALLEL_LEVEL=10 GIT_SHA=${{ github.sha }} PKG_CONFIG_PATH="$(pwd)/ffmpeg_${{ matrix.macos_arch }}/lib/pkgconfig" CMAKE_ARGS="-DHAVE_STD_REGEX=ON -DRUN_HAVE_STD_REGEX=1" | |
| # CIBW_BUILD: ${{ matrix.py_ver_prefix }}-* | |
| # CIBW_BUILD_VERBOSITY: 1 | |
| # CIBW_ARCHS_MACOS: ${{ matrix.macos_arch }} | |
| # CIBW_BEFORE_ALL_MACOS: brew list python@3.$(echo ${{ matrix.py_ver_prefix }} | cut -c 4-); if [ $? -eq 0 ]; then brew uninstall --ignore-dependencies python@3.$(echo ${{ matrix.py_ver_prefix }} | cut -c 4-); fi | |
| # CIBW_BEFORE_BUILD: > | |
| # ./scripts/build_ffmpeg.sh --arch=${{ matrix.macos_arch }} && | |
| # for xcode_python_root in /Applications/Xcode*/Contents/Developer/Library/Frameworks/Python3.framework;do if [ -d ${xcode_python_root} ]; then mv ${xcode_python_root} ${xcode_python_root}.bak; fi; done | |
| # CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" | |
| # - uses: actions/upload-artifact@v4 | |
| # with: | |
| # name: Build macos wheels with ${{ matrix.py_ver_prefix }}-${{ matrix.macos_arch }} | |
| # path: ./wheelhouse/*.whl | |
| # - name: Setup tmate session | |
| # if: ${{ failure() }} | |
| # uses: mxschmitt/action-tmate@v3 | |
| upload_all_to_github_releases: | |
| needs: build_manylinux_wheels | |
| runs-on: ubuntu-latest | |
| if: startsWith(github.ref, 'refs/tags/') | |
| permissions: | |
| contents: write | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: dist | |
| pattern: Build* | |
| merge-multiple: true | |
| - name: Upload wheels to GitHub Release | |
| uses: softprops/action-gh-release@v1 | |
| with: | |
| files: dist/*.whl |