Skip to content

Commit e853abf

Browse files
kalverraerikburt
andauthored
[DX-3383] Optimize Docker Caching (#1487)
* cache-scope and max mode * Fix * Setup debugging * Remove extra go mod download * Optimize Docker caching * Optimize Docker caching * Cleanup * Verify image * Add image inspection * Remove DEBUG * Comments * tee it up * Plugin caching + debug * Fixes * Fix encoding * Changeset * Comments * pr debug * Don't cache plugins on PRs * Update actions * Caching * Test with just docker cache --------- Co-authored-by: Erik Burton <erikburt@users.noreply.github.com>
1 parent 4fd12c6 commit e853abf

7 files changed

Lines changed: 65 additions & 103 deletions

File tree

.changeset/proud-owls-lick.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"ctf-build-image": minor
3+
---
4+
5+
Adds inputs and tweaks docker caching strategy for better performance

.github/workflows/run-e2e-tests.yml

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -427,7 +427,7 @@ jobs:
427427
persist-credentials: false
428428

429429
- name: Setup Go
430-
uses: actions/setup-go@v5.0.2
430+
uses: actions/setup-go@v6
431431
with:
432432
go-version: "1.24.0"
433433
check-latest: true
@@ -955,13 +955,13 @@ jobs:
955955
- name: Upload trace data as artifact
956956
if: inputs.enable_otel_traces_for_ocr2_plugins &&
957957
matrix.tests.test_env_vars.ENABLE_OTEL_TRACES == 'true'
958-
uses: actions/upload-artifact@v4.4.3
958+
uses: actions/upload-artifact@v7
959959
with:
960960
name: trace-data
961961
path: ./integration-tests/smoke/traces/trace-data.json
962962

963963
- name: Upload test log as artifact
964-
uses: actions/upload-artifact@v4.4.3
964+
uses: actions/upload-artifact@v7
965965
if: failure()
966966
with:
967967
name: test_log_${{ env.TEST_ID }}
@@ -971,7 +971,7 @@ jobs:
971971

972972
- name: Upload cl node coverage data as artifact
973973
if: inputs.upload_cl_node_coverage_artifact
974-
uses: actions/upload-artifact@v4.4.3
974+
uses: actions/upload-artifact@v7
975975
timeout-minutes: 2
976976
continue-on-error: true
977977
with:
@@ -988,7 +988,7 @@ jobs:
988988
989989
- name: Upload test result as artifact
990990
if: ${{ always() }}
991-
uses: actions/upload-artifact@v4.4.3
991+
uses: actions/upload-artifact@v7
992992
with:
993993
name: test_result_${{ needs.load-test-configurations.outputs.workflow_id
994994
}}_${{ env.TEST_ID }}
@@ -997,7 +997,7 @@ jobs:
997997

998998
- name: Upload custom test artifacts
999999
if: failure() && matrix.tests.test_artifacts_on_failure != ''
1000-
uses: actions/upload-artifact@v4.4.3
1000+
uses: actions/upload-artifact@v7
10011001
with:
10021002
name: custom_test_artifacts_${{ env.TEST_ID }}_${{
10031003
needs.load-test-configurations.outputs.workflow_id }}
@@ -1256,7 +1256,7 @@ jobs:
12561256
test_suite: ${{ matrix.tests.test_env_vars.TEST_SUITE }}
12571257

12581258
- name: Upload test log as artifact
1259-
uses: actions/upload-artifact@v4.4.3
1259+
uses: actions/upload-artifact@v7
12601260
if: failure()
12611261
with:
12621262
name: test_log_${{ env.TEST_ID }}
@@ -1266,7 +1266,7 @@ jobs:
12661266

12671267
- name: Upload custom test artifacts
12681268
if: failure() && matrix.tests.test_artifacts_on_failure != ''
1269-
uses: actions/upload-artifact@v4.4.3
1269+
uses: actions/upload-artifact@v7
12701270
with:
12711271
name: ${{ format('custom_test_artifacts_{0}_{1}', env.TEST_ID, needs.load-test-configurations.outputs.workflow_id) }}
12721272
path: ${{ matrix.tests.test_artifacts_on_failure }}

.github/workflows/solidity-review-artifacts.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,7 @@ jobs:
184184
done
185185
186186
- name: Upload basic info and modified contracts list
187-
uses: actions/upload-artifact@v4.6.2
187+
uses: actions/upload-artifact@v7
188188
timeout-minutes: 2
189189
continue-on-error: true
190190
with:
@@ -284,7 +284,7 @@ jobs:
284284
inputs.foundry_profile_override || inputs.product }}
285285

286286
- name: Upload Artifacts for product contracts
287-
uses: actions/upload-artifact@v4.6.2
287+
uses: actions/upload-artifact@v7
288288
timeout-minutes: 2
289289
continue-on-error: true
290290
with:
@@ -401,7 +401,7 @@ jobs:
401401
./dot_github/tools/scripts/solidity/generate_slither_report.sh "${{ github.server_url }}/${{ github.repository }}/blob/${{ env.head_ref }}/" "$SLITHER_CONFIG_FILE_PATH" "$CONTRACTS_DIRECTORY" "$contract_list" "${{ env.artifacts_dir }}/slither-reports" "--solc-remaps @=$CONTRACTS_DIRECTORY/node_modules/@"
402402
403403
- name: Upload UMLs and Slither reports
404-
uses: actions/upload-artifact@v4.6.2
404+
uses: actions/upload-artifact@v7
405405
timeout-minutes: 10
406406
continue-on-error: true
407407
with:
@@ -437,7 +437,7 @@ jobs:
437437
merge-multiple: true
438438

439439
- name: Upload all artifacts as single package
440-
uses: actions/upload-artifact@v4.6.2
440+
uses: actions/upload-artifact@v7
441441
with:
442442
name: review-artifacts-${{ inputs.product }}-${{ inputs.base_ref }}-${{
443443
env.head_ref }}

actions/apidiff-go/README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ jobs:
6767
fetch-depth: 0
6868

6969
- name: Set up Go
70-
uses: actions/setup-go@v5
70+
uses: actions/setup-go@v6
7171
with:
7272
go-version-file: "go.mod"
7373
cache: false
@@ -140,7 +140,7 @@ jobs:
140140
fetch-depth: 0
141141
142142
- name: Set up Go
143-
uses: actions/setup-go@v5
143+
uses: actions/setup-go@v6
144144
# uses: ./.github/actions/setup-go
145145
with:
146146
go-version-file: ${{matrix.modules}}/go.mod

actions/build-push-docker/action.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -203,14 +203,14 @@ runs:
203203

204204
- name: Login to private ECR registries for base images
205205
if: ${{ steps.dockerfile-ecr-parse.outputs.needs-ecr-login == 'true' }}
206-
uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
206+
uses: aws-actions/amazon-ecr-login@183a1442edf41672e66566b7fc560e297a290896 # v2.1.1
207207
with:
208208
registries: ${{ steps.dockerfile-ecr-parse.outputs.ecr-registries }}
209209

210210
- name: Login to ECR for publishing
211211
if: ${{ inputs.docker-push == 'true' }}
212212
id: login-ecr
213-
uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
213+
uses: aws-actions/amazon-ecr-login@183a1442edf41672e66566b7fc560e297a290896 # v2.1.1
214214
with:
215215
registry-type: >-
216216
${{
@@ -221,13 +221,13 @@ runs:
221221
registries: ${{ inputs.aws-account-number }}
222222

223223
- name: Set up Docker Buildx
224-
uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v3.9.0
224+
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
225225
with:
226226
version: latest
227227

228228
- name: Docker meta
229229
id: docker-meta
230-
uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
230+
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
231231
with:
232232
images:
233233
${{ format('{0}/{1}', inputs.docker-registry-url,

actions/ctf-build-image/action.yml

Lines changed: 42 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,21 @@ inputs:
108108
go get github.com/smartcontractkit/chainlink-solana@abc123
109109
go get github.com/smartcontractkit/chainlink-evm@def456
110110
111+
cache-scope:
112+
required: false
113+
description: |
114+
Custom scope for Docker build cache. Separates caches when multiple
115+
Dockerfiles build on the same architecture (e.g. "core" vs "plugins").
116+
If not set, defaults to runner OS and architecture.
117+
default: ""
118+
119+
free-disk-space:
120+
required: false
121+
description: |
122+
Whether to run the free-disk-space step before building. Set to "false"
123+
on runners with sufficient disk (e.g. RunsOn with 100GB+) to save ~30-60s.
124+
default: "true"
125+
111126
outputs:
112127
docker-image-sha-digest-amd64:
113128
description: "Docker image SHA digest for platform: amd64"
@@ -119,12 +134,6 @@ outputs:
119134
runs:
120135
using: composite
121136
steps:
122-
- uses: actions/setup-go@v6
123-
with:
124-
go-version-file: "go.mod"
125-
check-latest: true
126-
cache: false
127-
128137
- name: Setup GitHub token using GATI
129138
if: inputs.gati-role-arn != '' && inputs.gati-lambda-url != ''
130139
id: github-token
@@ -136,13 +145,6 @@ runs:
136145
aws-role-duration-seconds: "1800"
137146
set-git-config: "true"
138147

139-
- name: Process go get overrides
140-
shell: bash
141-
env:
142-
GO_OVERRIDES: ${{ inputs.go-get-overrides }}
143-
ACTIONS_PATH: ${{ github.action_path }}
144-
run: ${ACTIONS_PATH}/scripts/go-get-overrides.sh
145-
146148
- name: Process plugin manifest overrides (public)
147149
shell: bash
148150
env:
@@ -152,18 +154,32 @@ runs:
152154
ACTIONS_PATH: ${{ github.action_path }}
153155
run: ${ACTIONS_PATH}/scripts/plugin-overrides.sh
154156

155-
- name: Tidy and Output go.mod
157+
- name: Setup Go for dependency overrides
158+
if: inputs.go-get-overrides != ''
159+
uses: actions/setup-go@v6
160+
with:
161+
go-version-file: go.mod
162+
cache: false
163+
164+
- name: Apply go-get dependency overrides
165+
if: inputs.go-get-overrides != ''
156166
shell: bash
167+
env:
168+
GO_OVERRIDES: ${{ inputs.go-get-overrides }}
157169
run: |
158-
echo "::group::Tidy go.mod"
170+
set -e
171+
while IFS= read -r line; do
172+
[ -z "$line" ] && continue
173+
dep="${line%%=*}"
174+
sha="${line#*=}"
175+
[ -z "$dep" ] || [ -z "$sha" ] && continue
176+
echo "Overriding: github.com/smartcontractkit/${dep}@${sha}"
177+
go get "github.com/smartcontractkit/${dep}@${sha}"
178+
done <<< "$GO_OVERRIDES"
159179
go mod tidy
160-
echo "::endgroup::"
161-
162-
echo "::group::cat go.mod"
163-
cat go.mod
164-
echo "::endgroup::"
165180
166181
- name: Free up disk space (to avoid 'no space left on device' errors)
182+
if: inputs.free-disk-space == 'true'
167183
uses: smartcontractkit/.github/actions/free-disk-space@free-disk-space/v1
168184

169185
- name: Build push docker image
@@ -182,18 +198,18 @@ runs:
182198
docker-registry-url: ${{ inputs.docker-registry-url }}
183199
docker-repository-name: ${{ inputs.docker-repository-name }}
184200
# only save on events which are expected to be from the default branch
185-
docker-save-cache:
186-
${{ github.event_name == 'schedule' || github.event_name == 'push' }}
201+
docker-save-cache: ${{ github.event_name == 'schedule' ||
202+
github.event_name == 'push' || github.event_name == 'pull_request' }} # TODO: Remove pull_request after testing
187203
# dont use cache on events which are expected to be from the default branch
188204
# this is to create a fresh cache/snapshot unpolluted by previous cache entries
189205
docker-restore-cache:
190206
${{ github.event_name != 'schedule' && github.event_name != 'push' }}
191207
docker-build-cache-to:
192-
"type=gha,timeout=10m,mode=min,ignore-error=true,scope=ctf-build-image-${{
193-
runner.os }}-${{ runner.arch }}"
208+
"type=gha,timeout=10m,mode=max,ignore-error=true,compression=zstd,compression-level=3,scope=ctf-build-image-${{
209+
inputs.cache-scope || format('{0}-{1}', runner.os, runner.arch) }}"
194210
docker-build-cache-from:
195-
"type=gha,timeout=10m,scope=ctf-build-image-${{ runner.os }}-${{
196-
runner.arch }}"
211+
"type=gha,timeout=10m,scope=ctf-build-image-${{ inputs.cache-scope ||
212+
format('{0}-{1}', runner.os, runner.arch) }}"
197213

198214
tags: type=raw,value=${{ inputs.image-tag }}
199215
aws-account-number: ${{ inputs.aws-account-number }}

actions/ctf-build-image/scripts/go-get-overrides.sh

Lines changed: 0 additions & 59 deletions
This file was deleted.

0 commit comments

Comments
 (0)