From 07695f7dd048aa6f6730f7fdd4f99a169ef2c1ef Mon Sep 17 00:00:00 2001 From: Lindsay Holmwood Date: Wed, 28 Oct 2026 23:18:30 +1100 Subject: [PATCH 1/3] ci(runners): run the x64 Blacksmith jobs on RunsOn Blacksmith costs too much for this repository's CI. From 22 to 28 October 2026, these jobs used 8,841 Blacksmith minutes at 16 vCPUs. That cost $283 for the week. CipherStash keeps CI spending for all its repositories under $1,000 a month. RunsOn starts a machine in our own AWS account for each job. Heavy jobs across CipherStash move to it, with the same machines and labels: - family=c8a+m8a: an AMD c8a machine, or an m8a machine when AWS has no spare c8a spot capacity. - cpu=8: 8 vCPUs. - image: the Ubuntu version the Blacksmith label named (22.04 or 24.04). - volume=80gb: room for a release build of the workspace. - extras=s3-cache: the Actions cache is kept in an S3 bucket in the same AWS account. The runner reaches it faster than GitHub's cache. This commit changes the runner labels only. Job names, steps, and timeouts are unchanged. One exception: the name of the release build for linux/amd64 shows its runner label, so that name changes. The arm64 manifest job keeps its Blacksmith arm runner until we decide where arm64 jobs run. --- .github/workflows/benchmark.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ce31d2d38..c622f0934 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -26,7 +26,7 @@ permissions: jobs: benchmark: name: Performance regression check - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache env: CS_ZEROKMS_HOST: https://us-east-1.aws.zerokms.cipherstashmanaged.net CS_CTS_HOST: https://ap-southeast-2.aws.cts.cipherstashmanaged.net diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6760ad776..7a83e020e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,7 +52,7 @@ jobs: fail-fast: false matrix: build: - - { os: blacksmith-16vcpu-ubuntu-2404, docker_platform: linux/amd64, rust_target: "x86_64-unknown-linux-gnu" } + - { os: "runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu24-full-x64/volume=80gb/extras=s3-cache", docker_platform: linux/amd64, rust_target: "x86_64-unknown-linux-gnu" } - { os: linux-arm64-public, docker_platform: linux/arm64, rust_target: "aarch64-unknown-linux-gnu" } env: CS_ZEROKMS_HOST: https://us-east-1.aws.zerokms.cipherstashmanaged.net diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6bc55a6f1..2b51acec9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,7 +22,7 @@ env: jobs: test: name: Test (PostgreSQL ${{ matrix.pg_version }}) - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache strategy: fail-fast: false matrix: @@ -74,7 +74,7 @@ jobs: # change that raises the compiler requirement fails here. msrv: name: Build (Rust MSRV) - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache timeout-minutes: 30 steps: @@ -124,7 +124,7 @@ jobs: # adds Clippy lints, before the pinned toolchain is bumped to it. stable: name: Lint (Rust stable) - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache timeout-minutes: 30 env: # Overrides rust-toolchain.toml. @@ -154,7 +154,7 @@ jobs: burn-in: name: Encrypted burn-in (PostgreSQL 17) - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache timeout-minutes: 15 env: PG_VERSION: 17 @@ -200,7 +200,7 @@ jobs: pooler-compatibility: name: Supported pooler compatibility (PostgreSQL 17) - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache timeout-minutes: 45 env: PG_VERSION: 17 From 66e27b7da492242b502939669086df5928947ab7 Mon Sep 17 00:00:00 2001 From: Lindsay Holmwood Date: Thu, 29 Oct 2026 00:26:12 +1100 Subject: [PATCH 2/3] ci(runners): publish the multi-platform image from a RunsOn x64 machine "Publish multi-platform image" was the last job on a Blacksmith machine, an arm64 one with 16 vCPUs. It only joins the two per-platform image digests into one manifest list with `docker buildx imagetools create`. That command does not depend on the machine's architecture. The job now uses the same RunsOn x64 label as the other jobs in this repository. No job here runs on Blacksmith any more. --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7a83e020e..010c85a4d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -138,7 +138,7 @@ jobs: merge: name: Publish multi-platform image - runs-on: blacksmith-16vcpu-ubuntu-2404-arm + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu24-full-x64/volume=80gb/extras=s3-cache timeout-minutes: 30 needs: - build From dba896944082d89d024bc3bf36dcd2996a169889 Mon Sep 17 00:00:00 2001 From: Lindsay Holmwood Date: Thu, 29 Oct 2026 20:42:27 +1100 Subject: [PATCH 3/3] ci(docker): log in to Docker Hub before pulling images On RunsOn machines, Docker Hub refused anonymous image pulls with "429 Too Many Requests" and "toomanyrequests". Docker Hub limits anonymous pulls for each IP address. The test jobs failed while pulling postgres:17 and ubuntu:25.10. Every job that pulls from Docker Hub now logs in first. It uses the same credentials and action that release.yml already uses: the DOCKER_HUB_USERNAME and DOCKER_HUB_PASSWORD values from .github/secrets.env.encrypted, and docker/login-action. The login step comes straight after "Decrypt secrets" in these jobs: - Test (PostgreSQL 14 to 17) - Encrypted burn-in (PostgreSQL 17) - Supported pooler compatibility (PostgreSQL 17) - Performance regression check In release.yml, the build job already logged in, but only after `mise run build`. That task builds the Docker image, so it pulled ubuntu:25.10 before the login. The login step moves up to just after "Decrypt secrets". Build (Rust MSRV) and Lint (Rust stable) pull no images, so they do not log in. --- .github/workflows/benchmark.yml | 6 ++++++ .github/workflows/release.yml | 13 +++++++------ .github/workflows/test.yml | 18 ++++++++++++++++++ 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c622f0934..1aefe3307 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -46,6 +46,12 @@ jobs: CS_CLIENT_ACCESS_KEY: ${{ secrets.CS_VAULT_CLIENT_ACCESS_KEY }} CS_WORKSPACE_CRN: ${{ secrets.CS_VAULT_WORKSPACE_CRN }} + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_HUB_USERNAME }} + password: ${{ env.DOCKER_HUB_PASSWORD }} + - run: | mise run postgres:up --extra-args "--detach --wait" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 010c85a4d..223f7decd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -73,6 +73,13 @@ jobs: CS_CLIENT_ACCESS_KEY: ${{ secrets.CS_VAULT_CLIENT_ACCESS_KEY }} CS_WORKSPACE_CRN: ${{ secrets.CS_VAULT_WORKSPACE_CRN }} + # Log in before `mise run build`. It pulls the image's base from Docker Hub. + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_HUB_USERNAME }} + password: ${{ env.DOCKER_HUB_PASSWORD }} + - name: Setup Rust cache uses: Swatinem/rust-cache@v2 if: github.event_name == 'pull_request' # only cache in pull requests @@ -102,12 +109,6 @@ jobs: with: images: ${{ env.REGISTRY_IMAGE }} - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ env.DOCKER_HUB_USERNAME }} - password: ${{ env.DOCKER_HUB_PASSWORD }} - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2b51acec9..44ea8739d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,6 +47,12 @@ jobs: CS_CLIENT_ACCESS_KEY: ${{ secrets.CS_VAULT_CLIENT_ACCESS_KEY }} CS_WORKSPACE_CRN: ${{ secrets.CS_VAULT_WORKSPACE_CRN }} + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_HUB_USERNAME }} + password: ${{ env.DOCKER_HUB_PASSWORD }} + - run: | mise run postgres:up --extra-args "--detach --wait" @@ -176,6 +182,12 @@ jobs: CS_CLIENT_ACCESS_KEY: ${{ secrets.CS_VAULT_CLIENT_ACCESS_KEY }} CS_WORKSPACE_CRN: ${{ secrets.CS_VAULT_WORKSPACE_CRN }} + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_HUB_USERNAME }} + password: ${{ env.DOCKER_HUB_PASSWORD }} + - name: Start bare PostgreSQL and download EQL run: | mise run postgres:up --extra-args "--detach --wait" @@ -223,6 +235,12 @@ jobs: CS_CLIENT_ACCESS_KEY: ${{ secrets.CS_VAULT_CLIENT_ACCESS_KEY }} CS_WORKSPACE_CRN: ${{ secrets.CS_VAULT_WORKSPACE_CRN }} + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_HUB_USERNAME }} + password: ${{ env.DOCKER_HUB_PASSWORD }} + - name: Verify gate rejects incomplete runs and induced regressions run: python3 tests/tasks/test/pooler_gate_test.py