From 38510947d771383d99859bcc59725270911137cf Mon Sep 17 00:00:00 2001 From: Josh Cunningham Date: Wed, 16 Jul 2025 14:01:58 -0500 Subject: [PATCH 1/2] simplify actions with buildcloud --- .../build-and-push/action.yaml | 76 -------- .../build-and-test/action.yaml | 94 ---------- .github/action_templates/slack/action.yaml | 29 --- .github/actions/docker-build/action.yml | 64 +++++++ .github/workflows/docker_build_pr.yml | 27 +++ .../workflows/docker_image_main_branch.yml | 176 ------------------ .github/workflows/docker_publish_release.yml | 27 +++ 7 files changed, 118 insertions(+), 375 deletions(-) delete mode 100644 .github/action_templates/build-and-push/action.yaml delete mode 100644 .github/action_templates/build-and-test/action.yaml delete mode 100644 .github/action_templates/slack/action.yaml create mode 100644 .github/actions/docker-build/action.yml create mode 100644 .github/workflows/docker_build_pr.yml delete mode 100644 .github/workflows/docker_image_main_branch.yml create mode 100644 .github/workflows/docker_publish_release.yml diff --git a/.github/action_templates/build-and-push/action.yaml b/.github/action_templates/build-and-push/action.yaml deleted file mode 100644 index 988221a8..00000000 --- a/.github/action_templates/build-and-push/action.yaml +++ /dev/null @@ -1,76 +0,0 @@ -name: build and push template -description: "A template to build and push docker image based on provided inputs" - -inputs: - docker-auth-token: - description: 'Docker Hub authentication token' - required: true - image-name: - description: 'image name to be built' - required: true - dockerfile-name: - description: 'The name of dockerfile to use' - required: true - platform: - description: 'The platform to build' - required: false - -outputs: - tag-name: - description: "The tag name used for the image" - value: ${{ steps.set-outputs.outputs.tag-name }} - -runs: - using: "composite" - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Set env vars for main branch - shell: bash - run: | - echo "TAG_NAME=latest" >> "$GITHUB_ENV" - if: startsWith(github.ref, 'refs/heads/main') - - name: Set env vars for other branch - shell: bash - run: | - echo "TAG_NAME=${GITHUB_REF##*/}" >> "$GITHUB_ENV" - if: ${{ !startsWith(github.ref, 'refs/heads/main') }} - - - name: Set outputs - id: set-outputs - shell: bash - run: | - echo "Current TAG_NAME from env: ${{ env.TAG_NAME }}" - echo "tag-name=${{ env.TAG_NAME }}" >> $GITHUB_OUTPUT - - - name: Attach platform to tag - shell: bash - run: | - echo "TAG_NAME=${{env.TAG_NAME}}-${{inputs.platform}}" >> "$GITHUB_ENV" - - - name: Login to Docker Hub - shell: bash - run: | - echo "${{ inputs.docker-auth-token }}" | docker login --username awiciroh --password-stdin - - - name: Create Cache and Builder for Buildx - shell: bash - run: | - docker buildx create --use --name mybuilder --node mybuilder - - - name: Build and Cache Docker Image - uses: docker/build-push-action@v5 - with: - context: ./docker - file: ./docker/${{ inputs.dockerfile-name}} - push: true - tags: | - awiciroh/${{ inputs.image-name}}:${{ env.TAG_NAME}} - builder: mybuilder - no-cache: true - platforms: linux/${{ inputs.platform }} - provenance: false - env: - DOCKER_BUILDKIT: 1 - DOCKER_CLI_EXPERIMENTAL: enabled \ No newline at end of file diff --git a/.github/action_templates/build-and-test/action.yaml b/.github/action_templates/build-and-test/action.yaml deleted file mode 100644 index 3a129a22..00000000 --- a/.github/action_templates/build-and-test/action.yaml +++ /dev/null @@ -1,94 +0,0 @@ -name: build and test template -description: "A template to build and test docker image based on provided inputs" - -inputs: - docker-auth-token: - description: 'Docker Hub authentication token' - required: true - image-name: - description: 'image name to be built' - required: true - dockerfile-name: - description: 'The name of dockerfile to use' - required: true - platform: - description: 'The platform to build' - required: false - -outputs: - tag-name: - description: "The tag name used for the image" - value: ${{ steps.set-outputs.outputs.tag-name }} - -runs: - using: "composite" - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Set env vars for main branch - shell: bash - run: | - echo "TAG_NAME=latest" >> "$GITHUB_ENV" - if: startsWith(github.ref, 'refs/heads/main') - - name: Set env vars for other branch - shell: bash - run: | - echo "TAG_NAME=${GITHUB_REF##*/}" >> "$GITHUB_ENV" - if: ${{ !startsWith(github.ref, 'refs/heads/main') }} - - - name: Set outputs - id: set-outputs - shell: bash - run: | - echo "Current TAG_NAME from env: ${{ env.TAG_NAME }}" - echo "tag-name=${{ env.TAG_NAME }}" >> $GITHUB_OUTPUT - - - name: Attach platform to tag - shell: bash - run: | - echo "TAG_NAME=${{env.TAG_NAME}}-${{inputs.platform}}" >> "$GITHUB_ENV" - - - name: Login to Docker Hub - shell: bash - run: | - echo "Logging in to Docker Hub..." - if ! echo "${{ inputs.docker-auth-token }}" | docker login --username awiciroh --password-stdin; then - echo "Failed to login to Docker Hub" - exit 1 - fi - echo "Successfully logged in to Docker Hub" - docker info - - - name: Create Cache and Builder for Buildx - shell: bash - run: | - docker buildx create --use --name mybuilder --node mybuilder - - - name: Build and Cache Docker Image - uses: docker/build-push-action@v5 - with: - context: ./docker - file: ./docker/${{ inputs.dockerfile-name}} - push: true - tags: | - awiciroh/${{ inputs.image-name}}:${{ env.TAG_NAME}} - builder: mybuilder - no-cache: true - platforms: linux/${{ inputs.platform }} - provenance: false - env: - DOCKER_BUILDKIT: 1 - DOCKER_CLI_EXPERIMENTAL: enabled - - name: Test - shell: bash - run: | - rm -rf /home/ubuntu/workspace/AWI_16_2863657_007/*.csv - rm -rf /home/ubuntu/workspace/AWI_16_2863657_007/*.parquet - rm -rf /home/ubuntu/workspace/AWI_16_2863657_007/outputs/*/* - rm -rf /home/ubuntu/workspace/AWI_16_2863806_008/*.csv - rm -rf /home/ubuntu/workspace/AWI_16_2863806_008/*.parquet - rm -rf /home/ubuntu/workspace/AWI_16_2863806_008/outputs/*/* - docker run --rm -v /home/ubuntu/workspace/AWI_16_2863657_007:/ngen/ngen/data awiciroh/${{ inputs.image-name}}:${{ env.TAG_NAME}} /ngen/ngen/data auto - docker run --rm -v /home/ubuntu/workspace/AWI_16_2863806_008:/ngen/ngen/data awiciroh/${{ inputs.image-name}}:${{ env.TAG_NAME}} /ngen/ngen/data auto - if: inputs.image-name == 'ciroh-ngen-image' diff --git a/.github/action_templates/slack/action.yaml b/.github/action_templates/slack/action.yaml deleted file mode 100644 index 8d038569..00000000 --- a/.github/action_templates/slack/action.yaml +++ /dev/null @@ -1,29 +0,0 @@ -name: Send Slack Message -description: "Send slack message to a channel" - -inputs: - slack-webhook-url: - description: 'Slack webhook url' - required: true - job-status: - description: 'Job status' - required: true - pull_request_url: - description: 'Pull request url' - required: true - head_commit_url: - description: 'Head commit url' - required: true - -runs: - using: "composite" - steps: - - name: send slack message - uses: slackapi/slack-github-action@v1.24.0 - with: - payload: | - { - "text": "GitHub Action build result: ${{ inputs.job-status }}\n${{ inputs.pull_request_url || inputs.head_commit_url }}" - } - env: - SLACK_WEBHOOK_URL: ${{ inputs.slack-webhook-url}} \ No newline at end of file diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml new file mode 100644 index 00000000..9f629853 --- /dev/null +++ b/.github/actions/docker-build/action.yml @@ -0,0 +1,64 @@ +name: 'Docker Build and Push' +description: 'Build and push Docker images with Slack notification' + +inputs: + dockerfile-tag: + description: 'Tag to replace in Dockerfile' + required: true + docker-tags: + description: 'Newline-separated list of Docker tags' + required: true + slack-title: + description: 'Title for Slack notification' + required: true + dockerhub-username: + description: 'DockerHub username' + required: true + dockerhub-token: + description: 'DockerHub token' + required: true + buildcloud-endpoint: + description: 'Buildcloud endpoint' + required: true + slack-webhook-url: + description: 'Slack webhook URL' + required: true + +runs: + using: "composite" + steps: + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ inputs.dockerhub-username }} + password: ${{ inputs.dockerhub-token }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + version: "lab:latest" + driver: cloud + endpoint: ${{ inputs.buildcloud-endpoint }} + + - name: Update Dockerfile + shell: bash + run: sed -i "s/@ngiab_dev/@ngiab_$(echo ${{ inputs.dockerfile-tag }} | sed 's/\./\\./g')/g" ./docker/Dockerfile + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: ./docker + file: ./docker/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ inputs.docker-tags }} + + - name: Slack Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_CHANNEL: ngen-github-ci + SLACK_COLOR: ${{ job.status }} + SLACK_ICON: https://avatars.githubusercontent.com/u/119541160?s=48&v=4 + SLACK_TITLE: ${{ inputs.slack-title }} + SLACK_USERNAME: Github Action Result + SLACK_WEBHOOK: ${{ inputs.slack-webhook-url }} diff --git a/.github/workflows/docker_build_pr.yml b/.github/workflows/docker_build_pr.yml new file mode 100644 index 00000000..9402fda5 --- /dev/null +++ b/.github/workflows/docker_build_pr.yml @@ -0,0 +1,27 @@ +name: Build Pull Request Test Image + +on: + pull_request: + branches: [ main ] + paths: + - docker/** + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4.2.2 + + - name: Build and Push Docker + uses: ./.github/actions/docker-build + with: + dockerfile-tag: ${{ github.event.pull_request.number }} + docker-tags: | + ${{ secrets.DOCKERHUB_USERNAME }}/ciroh-ngen-image-dev:${{ github.event.pull_request.number }} + ${{ secrets.DOCKERHUB_USERNAME }}/ngiab-dev:${{ github.event.pull_request.number }} + slack-title: Pull request built ${{ github.event.pull_request.title }} + dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} + buildcloud-endpoint: ${{ secrets.BUILDCLOUD_ENDPOINT }} + slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/docker_image_main_branch.yml b/.github/workflows/docker_image_main_branch.yml deleted file mode 100644 index c53e2bc5..00000000 --- a/.github/workflows/docker_image_main_branch.yml +++ /dev/null @@ -1,176 +0,0 @@ -name: Build and push final image -on: - push: - branches: [ main ] - paths-ignore: - - '.github/ISSUE_TEMPLATE/**' - - 'README.md' - - 'docs/**' - pull_request: - branches: [ main ] - paths-ignore: - - '.github/ISSUE_TEMPLATE/**' - - 'README.md' - - 'docs/**' - workflow_dispatch: - -jobs: - start-runner: - runs-on: ubuntu-latest - steps: - - name: start arm runner - uses: gagoar/invoke-aws-lambda@master - with: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - FunctionName: StartCirohRunner - if: startsWith(github.event.pull_request.head.repo.full_name, 'CIROH-UA') || - (startsWith(github.ref, 'refs/heads/main') && github.event_name != 'pull_request') - - - name: start x86 runner - uses: gagoar/invoke-aws-lambda@master - with: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - FunctionName: StartCirohX86Runner - if: startsWith(github.event.pull_request.head.repo.full_name, 'CIROH-UA') || - (startsWith(github.ref, 'refs/heads/main') && github.event_name != 'pull_request') - - - name: wait runner up - shell: bash - run: | - sleep 300 - if: startsWith(github.event.pull_request.head.repo.full_name, 'CIROH-UA') || - (startsWith(github.ref, 'refs/heads/main') && github.event_name != 'pull_request') - init: - needs: start-runner - runs-on: ${{ matrix.runner}} - strategy: - matrix: - runner: [ARM64, AMD64] - steps: - - name: clear docker cache - run: | - docker system prune -af - build: - needs: init - runs-on: ${{ matrix.runner}} - outputs: # Define job outputs - test-tag-name: ${{ steps.test-build.outputs.tag-name }} - push-tag-name: ${{ steps.push-build.outputs.tag-name }} - strategy: - matrix: - runner: [arm64, amd64] - steps: - - name: deploy - uses: actions/checkout@v2 - - name: Build and test Docker image - id: test-build - uses: ./.github/action_templates/build-and-test - with: - docker-auth-token: ${{ secrets.DOCKER_AUTH_TOKEN }} - image-name: "ciroh-ngen-image" - dockerfile-name: "Dockerfile" - platform: ${{ matrix.runner}} - if: ${{ !startsWith(github.ref, 'refs/heads/main') }} - - name: Build and push Docker image - id: push-build - uses: ./.github/action_templates/build-and-push - with: - docker-auth-token: ${{ secrets.DOCKER_AUTH_TOKEN }} - image-name: "ciroh-ngen-image" - dockerfile-name: "Dockerfile" - platform: ${{ matrix.runner}} - if: startsWith(github.ref, 'refs/heads/main') - check_result: - needs: build - runs-on: ${{ matrix.runner}} - strategy: - matrix: - runner: [ARM64, AMD64] - outputs: - output1: ${{ steps.step1.outputs.result }} - steps: - - id: step1 - run: echo "result=success" >> "$GITHUB_OUTPUT" - - create-manifest: - needs: [check_result, build] - runs-on: ARM64 - steps: - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: awiciroh - password: ${{ secrets.DOCKER_AUTH_TOKEN }} - - - name: Create and push manifest - run: | - if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then - echo "Using push tag (main branch)" - TAG_NAME="${{ needs.build.outputs.push-tag-name }}" - else - echo "Using test tag (non-main branch)" - TAG_NAME="${{ needs.build.outputs.test-tag-name }}" - if [[ -z "${TAG_NAME}" ]]; then - echo "Error: test tag is empty" - exit 1 - fi - fi - - echo "Creating manifest with TAG_NAME: ${TAG_NAME}" - - # Remove existing manifest if exists - docker manifest rm awiciroh/ciroh-ngen-image:${TAG_NAME} || true - - docker manifest create --amend awiciroh/ciroh-ngen-image:${TAG_NAME} \ - awiciroh/ciroh-ngen-image:${TAG_NAME}-arm64 \ - awiciroh/ciroh-ngen-image:${TAG_NAME}-amd64 - docker manifest push awiciroh/ciroh-ngen-image:${TAG_NAME} - - slack: - runs-on: ARM64 - if: startsWith(github.event.pull_request.head.repo.full_name, 'CIROH-UA') || - (startsWith(github.ref, 'refs/heads/main') && github.event_name != 'pull_request') - needs: [build, check_result,create-manifest] - env: - OUTPUT: ${{needs.check_result.outputs.output1}} - steps: - - name: Set env JOBSTATUS - run: | - echo "$OUTPUT" - if [[ "success" = "$OUTPUT" ]] - then - echo "JOBSTATUS=success" >> "$GITHUB_ENV" - else - echo "JOBSTATUS=failure" >> "$GITHUB_ENV" - fi - - run: echo "$JOBSTATUS" - - name: deploy - uses: actions/checkout@v2 - - name: Publish slack message - if: always() - uses: ./.github/action_templates/slack - with: - slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} - job-status: ${{ env.JOBSTATUS }} - pull_request_url: ${{ github.event.pull_request.html_url }} - head_commit_url: ${{ github.event.head_commit.url }} - stop-runner: - needs: slack - runs-on: ubuntu-latest - if: startsWith(github.event.pull_request.head.repo.full_name, 'CIROH-UA') || - (startsWith(github.ref, 'refs/heads/main') && github.event_name != 'pull_request') - steps: - - name: stop arm runner - uses: gagoar/invoke-aws-lambda@master - with: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - FunctionName: StopCirohRunner - - name: stop x86 runner - uses: gagoar/invoke-aws-lambda@master - with: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - FunctionName: StopCirohX86Runner diff --git a/.github/workflows/docker_publish_release.yml b/.github/workflows/docker_publish_release.yml new file mode 100644 index 00000000..c14aa029 --- /dev/null +++ b/.github/workflows/docker_publish_release.yml @@ -0,0 +1,27 @@ +name: Build and Push Docker Image + +on: + release: + types: [published] + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4.2.2 + + - name: Build and Push Docker + uses: ./.github/actions/docker-build + with: + dockerfile-tag: ${{ github.event.release.tag_name }} + docker-tags: | + ${{ secrets.DOCKERHUB_USERNAME }}/ciroh-ngen-image:${{ github.event.release.tag_name }} + ${{ secrets.DOCKERHUB_USERNAME }}/ciroh-ngen-image:latest + ${{ secrets.DOCKERHUB_USERNAME }}/ngiab:${{ github.event.release.tag_name }} + ${{ secrets.DOCKERHUB_USERNAME }}/ngiab:latest + slack-title: Build and Release + dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} + buildcloud-endpoint: ${{ secrets.BUILDCLOUD_ENDPOINT }} + slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} From 29ab7a98038aeb39994f00081f52959f73686973 Mon Sep 17 00:00:00 2001 From: Josh Cunningham Date: Wed, 16 Jul 2025 14:09:30 -0500 Subject: [PATCH 2/2] add action to update dockerhub description --- .github/workflows/dockerhub_readme.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/workflows/dockerhub_readme.yml diff --git a/.github/workflows/dockerhub_readme.yml b/.github/workflows/dockerhub_readme.yml new file mode 100644 index 00000000..b798b6f8 --- /dev/null +++ b/.github/workflows/dockerhub_readme.yml @@ -0,0 +1,12 @@ +name: Update Dockerhub Image Description + +on: + push: + paths: + - 'README.md' + - name: Update Docker Hub Description + uses: peter-evans/dockerhub-description@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: ${{ secrets.DOCKERHUB_USERNAME }}/ciroh-ngen-image