Fix #1417: RMST-462: turn zstd into dcz (#1418) #1712
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Publish Docker Containers | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| push: | |
| branches: | |
| - main | |
| release: | |
| types: | |
| - released | |
| permissions: | |
| contents: read | |
| id-token: write | |
| env: | |
| BUILDX_NO_DEFAULT_ATTESTATIONS: 1 # Reduce warnings from Docker Buildx | |
| GAR_LOCATION: us | |
| GAR_REPOSITORY: remote-settings-prod | |
| GCP_PROJECT_ID: moz-fx-remote-settings-prod | |
| jobs: | |
| # Let's figure out which container we have to build based on which files were changed. | |
| # On pull-requests that don't touch these files, we don't need to rebuild them. | |
| # Of course, on push to main or releases, we always build and publish to GAR/Dockerhub. | |
| changes: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| cronjobs: ${{ steps.filter.outputs.cronjobs }} | |
| browser_tests: ${{ steps.filter.outputs.browser_tests }} | |
| git_reader: ${{ steps.filter.outputs.git_reader }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 0 | |
| - id: filter | |
| if: github.event_name == 'pull_request' | |
| env: | |
| BASE: ${{ github.event.pull_request.base.sha }} | |
| HEAD: ${{ github.event.pull_request.head.sha }} | |
| run: | | |
| files=$(git diff --name-only "$BASE" "$HEAD") | |
| echo "Changed files:" | |
| echo "$files" | |
| check() { | |
| if echo "$files" | grep -qE "$2"; then | |
| echo "$1=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "$1=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| } | |
| check cronjobs '^.github/workflows/|^cronjobs/' | |
| check browser_tests '^.github/workflows/|^browser-tests/' | |
| check git_reader '^.github/workflows/|^git-reader/' | |
| # Make sure we don't deploy images with failing tests. | |
| # On pull-requests, tests are already ran in CI. | |
| tests: | |
| if: github.event_name != 'pull_request' | |
| uses: ./.github/workflows/test.yaml | |
| server_container: | |
| # Always build the server (since it depends on lots of files uv.lock, dockerfiles, kinto-slack, etc.) | |
| needs: tests | |
| if: ${{ !cancelled() && needs.tests.result != 'failure' }} | |
| env: | |
| DOCKERHUB_IMAGE_NAME: mozilla/remote-settings | |
| GAR_IMAGE_NAME: remote-settings | |
| runs-on: ubuntu-latest | |
| environment: build | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v4 | |
| - name: Enable multiplatform builds | |
| uses: docker/setup-buildx-action@v4 | |
| with: | |
| buildkitd-flags: "--debug" # Enable detailed logging | |
| - name: Set tag version | |
| run: echo "LATEST_TAG=$(git describe --tags --abbrev=4)" >> "$GITHUB_ENV" | |
| - name: Create version.json | |
| run: | | |
| # create a version.json per https://github.com/mozilla-services/Dockerflow/blob/main/docs/version_object.md | |
| printf '{\n "commit": "%s",\n "version": "%s",\n "source": "%s",\n "build": "%s"\n}\n' \ | |
| "$GITHUB_SHA" \ | |
| "$LATEST_TAG" \ | |
| "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" \ | |
| "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" > ./version.json | |
| # Show complete version.json for debugging | |
| cat ./version.json | |
| - name: Extract metadata for Google Artifact Registry | |
| id: metagar | |
| uses: docker/metadata-action@v6 | |
| with: | |
| flavor: | |
| # don't automatically tag with `latest`; we do this conditionally in the `tags` section | |
| latest=false | |
| images: | | |
| ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.GAR_REPOSITORY }}/${{ env.GAR_IMAGE_NAME }} | |
| tags: | | |
| type=raw,value=${{ env.LATEST_TAG }},enable=${{ github.event_name == 'push' }} | |
| type=sha,format=long,enable=${{ github.event_name == 'push' }} | |
| type=semver,pattern={{raw}},enable=${{ github.event_name == 'release' }} | |
| - name: Docker Metadata for Docker Hub | |
| id: metahub | |
| uses: docker/metadata-action@v6 | |
| with: | |
| flavor: | |
| # don't automatically tag with `latest`; we do this conditionally in the `tags` section | |
| latest=false | |
| images: | | |
| ${{ env.DOCKERHUB_IMAGE_NAME }} | |
| tags: | | |
| type=semver,pattern={{raw}},enable=${{ github.event_name == 'release' }} | |
| type=raw,value=latest,enable=${{ github.event_name == 'push' }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Authenticate on GCP | |
| id: gcp_auth | |
| uses: google-github-actions/auth@v3 | |
| with: | |
| token_format: access_token | |
| service_account: artifact-writer@${{ env.GCP_PROJECT_ID }}.iam.gserviceaccount.com | |
| workload_identity_provider: ${{ vars.GCPV2_GITHUB_WORKLOAD_IDENTITY_PROVIDER }} | |
| - name: Login to GAR | |
| if: github.event_name != 'pull_request' | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.GAR_LOCATION }}-docker.pkg.dev | |
| username: oauth2accesstoken | |
| password: ${{ steps.gcp_auth.outputs.access_token }} | |
| - name: Login to Docker Hub | |
| if: github.event_name != 'pull_request' | |
| uses: docker/login-action@v4 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Build and push to GAR | |
| if: ${{ github.event_name == 'push' }} | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: . | |
| file: RemoteSettings.Dockerfile | |
| sbom: true | |
| target: production # See multi-stage build | |
| push: true | |
| tags: ${{ steps.metagar.outputs.tags }} | |
| labels: ${{ steps.metagar.outputs.labels }} | |
| platforms: linux/amd64,linux/arm64 | |
| cache-from: type=gha,scope=server | |
| cache-to: type=gha,mode=max,scope=server | |
| - name: Wait for release base image | |
| if: ${{ github.event_name == 'release' }} | |
| env: | |
| BASE_IMG: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.GAR_REPOSITORY }}/remote-settings:sha-${{ github.sha }} | |
| run: | | |
| # Wait up to 20min for the base image (built by the push workflow above, on the same SHA). | |
| deadline=$((SECONDS + 1200)) | |
| until docker buildx imagetools inspect "$BASE_IMG" >/dev/null 2>&1; do | |
| if [ $SECONDS -ge $deadline ]; then | |
| echo "::error::Timed out waiting for base image $BASE_IMG" | |
| exit 1 | |
| fi | |
| echo "Base image not yet available, retrying in 30s..." | |
| sleep 30 | |
| done | |
| echo "Base image is available." | |
| - name: Build and push release to GAR | |
| if: ${{ github.event_name == 'release' }} | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: . | |
| file: Release.Dockerfile | |
| sbom: true | |
| push: true | |
| tags: ${{ steps.metagar.outputs.tags }} | |
| labels: ${{ steps.metagar.outputs.labels }} | |
| platforms: linux/amd64,linux/arm64 | |
| cache-from: type=gha,scope=server-release | |
| cache-to: type=gha,mode=max,scope=server-release | |
| build-args: BASE_IMG=${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.GAR_REPOSITORY}}/remote-settings:sha-${{ github.sha }} | |
| - name: Copy from Google Artifact Registry to Docker Hub | |
| if: github.event_name != 'pull_request' | |
| env: | |
| TAGS: | | |
| ${{ steps.metahub.outputs.tags }} | |
| SRC: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.GAR_REPOSITORY}}/remote-settings:${{ env.LATEST_TAG }} | |
| run: | | |
| for tag in $TAGS; do | |
| docker buildx imagetools create --tag "${tag}" "${SRC}" | |
| done | |
| - name: Notify DEVs of build failure | |
| if: failure() | |
| uses: slackapi/slack-github-action@v3.0.5 | |
| with: | |
| webhook: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| webhook-type: incoming-webhook | |
| payload: | | |
| text: "⚠️ Build of ${{ env.GAR_IMAGE_NAME }}:${{ env.LATEST_TAG }} failed. Please review logs and correct issues." | |
| cronjobs_container: | |
| needs: changes | |
| if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.cronjobs == 'true' }} | |
| env: | |
| GAR_IMAGE_NAME: remote-settings-core-cronjobs | |
| LATEST_TAG: "" # Set after checkout step | |
| runs-on: ubuntu-latest | |
| environment: build | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set tag version | |
| run: echo "LATEST_TAG=$(git describe --tags --abbrev=4)" >> "$GITHUB_ENV" | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@v6 | |
| with: | |
| images: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.GAR_REPOSITORY }}/${{ env.GAR_IMAGE_NAME }} | |
| tags: | | |
| type=raw,value=latest | |
| type=raw,value=${{ env.LATEST_TAG }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Authenticate on GCP | |
| id: gcp_auth | |
| uses: google-github-actions/auth@v3 | |
| with: | |
| token_format: access_token | |
| service_account: artifact-writer@${{ env.GCP_PROJECT_ID }}.iam.gserviceaccount.com | |
| workload_identity_provider: ${{ vars.GCPV2_GITHUB_WORKLOAD_IDENTITY_PROVIDER }} | |
| - name: Login to GAR | |
| if: github.event_name == 'push' | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.GAR_LOCATION }}-docker.pkg.dev | |
| username: oauth2accesstoken | |
| password: ${{ steps.gcp_auth.outputs.access_token }} | |
| - name: Build and push | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: cronjobs/ | |
| push: ${{ github.event_name == 'push' }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| cache-from: type=gha,scope=cronjobs | |
| cache-to: type=gha,mode=max,scope=cronjobs | |
| - name: Notify DEVs of build failure | |
| if: failure() | |
| uses: slackapi/slack-github-action@v3.0.5 | |
| with: | |
| webhook: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| webhook-type: incoming-webhook | |
| payload: | | |
| text: "⚠️ Build of ${{ env.GAR_IMAGE_NAME }}:${{ env.LATEST_TAG }} failed. Please review logs and correct issues." | |
| browser_test_container: | |
| needs: changes | |
| if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.browser_tests == 'true' }} | |
| env: | |
| GAR_IMAGE_NAME: remote-settings-browser-tests | |
| LATEST_TAG: "" # Set after checkout step | |
| runs-on: ubuntu-latest | |
| environment: build | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set tag version | |
| run: echo "LATEST_TAG=$(git describe --tags --abbrev=4)" >> "$GITHUB_ENV" | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@v6 | |
| with: | |
| images: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.GAR_REPOSITORY }}/${{ env.GAR_IMAGE_NAME }} | |
| tags: | | |
| type=raw,value=${{ env.LATEST_TAG }} | |
| type=raw,value=latest | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Authenticate on GCP | |
| id: gcp_auth | |
| uses: google-github-actions/auth@v3 | |
| with: | |
| token_format: access_token | |
| service_account: artifact-writer@${{ env.GCP_PROJECT_ID }}.iam.gserviceaccount.com | |
| workload_identity_provider: ${{ vars.GCPV2_GITHUB_WORKLOAD_IDENTITY_PROVIDER }} | |
| - name: Login to GAR | |
| if: ${{ github.event_name == 'push' }} | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.GAR_LOCATION }}-docker.pkg.dev | |
| username: oauth2accesstoken | |
| password: ${{ steps.gcp_auth.outputs.access_token }} | |
| - name: Build and push | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: browser-tests/ | |
| push: ${{ github.event_name == 'push' }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| cache-from: type=gha,scope=browser-tests | |
| cache-to: type=gha,mode=max,scope=browser-tests | |
| - name: Notify DEVs of build failure | |
| if: failure() | |
| uses: slackapi/slack-github-action@v3.0.5 | |
| with: | |
| webhook: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| webhook-type: incoming-webhook | |
| payload: | | |
| text: "⚠️ Build of ${{ env.GAR_IMAGE_NAME }}:${{ env.LATEST_TAG }} failed. Please review logs and correct issues." | |
| git_reader_container: | |
| needs: changes | |
| if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.git_reader == 'true' }} | |
| env: | |
| GAR_IMAGE_NAME: remote-settings-git-reader | |
| LATEST_TAG: "" # Set after checkout step | |
| runs-on: ubuntu-latest | |
| environment: build | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set tag version | |
| run: echo "LATEST_TAG=$(git describe --tags --abbrev=4)" >> "$GITHUB_ENV" | |
| - name: Create version.json | |
| run: | | |
| # create a version.json per https://github.com/mozilla-services/Dockerflow/blob/main/docs/version_object.md | |
| printf '{\n "commit": "%s",\n "version": "%s",\n "source": "%s",\n "build": "%s"\n}\n' \ | |
| "$GITHUB_SHA" \ | |
| "$LATEST_TAG" \ | |
| "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" \ | |
| "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" > ./git-reader/version.json | |
| # Show complete version.json for debugging | |
| cat ./git-reader/version.json | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@v6 | |
| with: | |
| images: | | |
| ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.GAR_REPOSITORY }}/${{ env.GAR_IMAGE_NAME }} | |
| tags: | | |
| type=raw,value=${{ env.LATEST_TAG }},enable=${{ github.event_name == 'push' }} | |
| type=sha,format=long,enable=${{ github.event_name == 'push' }} | |
| type=semver,pattern={{raw}},enable=${{ github.event_name == 'release' }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Authenticate on GCP | |
| id: gcp_auth | |
| uses: google-github-actions/auth@v3 | |
| with: | |
| token_format: access_token | |
| service_account: artifact-writer@${{ env.GCP_PROJECT_ID }}.iam.gserviceaccount.com | |
| workload_identity_provider: ${{ vars.GCPV2_GITHUB_WORKLOAD_IDENTITY_PROVIDER }} | |
| - name: Login to GAR | |
| if: ${{ github.event_name != 'pull_request' }} | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.GAR_LOCATION }}-docker.pkg.dev | |
| username: oauth2accesstoken | |
| password: ${{ steps.gcp_auth.outputs.access_token }} | |
| - name: Build and push | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: git-reader/ | |
| push: ${{ github.event_name != 'pull_request' }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| cache-from: type=gha,scope=git-reader | |
| cache-to: type=gha,mode=max,scope=git-reader | |
| - name: Notify DEVs of build failure | |
| if: failure() | |
| uses: slackapi/slack-github-action@v3.0.5 | |
| with: | |
| webhook: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| webhook-type: incoming-webhook | |
| payload: | | |
| text: "⚠️ Build of ${{ env.GAR_IMAGE_NAME }}:${{ env.LATEST_TAG }} failed. Please review logs and correct issues." |