Skip to content

Commit 2776248

Browse files
QuleafQuleaf
committed
cicd:experiment
1 parent 48ebef3 commit 2776248

2 files changed

Lines changed: 153 additions & 182 deletions

File tree

‎.github/actions/setup-rclone/action.yml‎

Lines changed: 137 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: 'Setup Rclone'
2-
description: 'Installs and configures rclone for S3 uploads/downloads without sudo, using $HOME/.cache/rclone for caching. Optionally downloads archive from S3.'
2+
description: 'Configures rclone for S3 operations using system module rclone/1.68.1. Supports downloading archives from S3 and uploading files to S3 with optimized settings.'
33
inputs:
44
access_key_id:
55
description: 'Access Key ID for S3'
@@ -31,10 +31,22 @@ inputs:
3131
description: 'Whether to load the downloaded archive into podman'
3232
required: false
3333
default: 'false'
34+
# Upload-specific inputs
35+
upload_mode:
36+
description: 'Whether to upload file to S3 (true/false)'
37+
required: false
38+
default: 'false'
39+
upload_file:
40+
description: 'Local file path to upload - required when upload_mode=true'
41+
required: false
42+
upload_file_type:
43+
description: 'Type of file being uploaded (archive/sif) for output naming'
44+
required: false
45+
default: 'archive'
3446
outputs:
35-
rclone_installed:
36-
description: 'Indicates if rclone was installed in this run'
37-
value: ${{ steps.check_rclone.outputs.installed }}
47+
rclone_loaded:
48+
description: 'Indicates if rclone module was loaded successfully'
49+
value: ${{ steps.load_rclone.outputs.loaded }}
3850
# Download-specific outputs
3951
archive_downloaded:
4052
description: 'Indicates if archive was downloaded'
@@ -51,56 +63,44 @@ outputs:
5163
image_tag:
5264
description: 'Image tag if loaded to podman'
5365
value: ${{ steps.load.outputs.image_tag }}
66+
# Upload-specific outputs
67+
file_uploaded:
68+
description: 'Indicates if file was uploaded to S3'
69+
value: ${{ steps.upload.outputs.uploaded }}
70+
upload_bucket:
71+
description: 'S3 bucket where file was uploaded'
72+
value: ${{ steps.upload.outputs.s3_bucket }}
73+
upload_path:
74+
description: 'S3 path of the uploaded file'
75+
value: ${{ steps.upload.outputs.s3_path }}
76+
upload_size:
77+
description: 'Size of the uploaded file'
78+
value: ${{ steps.upload.outputs.file_size }}
5479
runs:
5580
using: 'composite'
5681
steps:
57-
- name: Prepare cache path env
58-
id: envprep
82+
- name: Load rclone module
83+
id: load_rclone
5984
shell: bash
6085
run: |
6186
set -euo pipefail
62-
# Define cache directory under $HOME/.cache/rclone
63-
echo "RCLONE_CACHE_DIR=$HOME/.cache/rclone" >> $GITHUB_ENV
64-
echo "RCLONE_CACHE_BIN=$HOME/.cache/rclone/rclone" >> $GITHUB_ENV
65-
66-
- name: Check if rclone is available locally or in cache
67-
id: check_rclone
68-
shell: bash
69-
run: |
70-
set -euo pipefail
71-
if [ -x "./rclone" ]; then
72-
# rclone binary already present in the current directory
73-
echo "Found local ./rclone"
74-
echo "installed=true" >> $GITHUB_OUTPUT
75-
elif [ -x "${RCLONE_CACHE_BIN}" ]; then
76-
# Found cached binary, copy it to current directory
77-
echo "Found cached rclone at ${RCLONE_CACHE_BIN}; copying to ./rclone"
78-
cp "${RCLONE_CACHE_BIN}" ./rclone
79-
chmod +x ./rclone
80-
echo "installed=true" >> $GITHUB_OUTPUT
87+
echo "Loading rclone module..."
88+
if module load rclone/1.68.1; then
89+
echo "✓ rclone module loaded successfully"
90+
echo "loaded=true" >> $GITHUB_OUTPUT
91+
# Verify rclone is available
92+
if command -v rclone >/dev/null 2>&1; then
93+
echo "✓ rclone command is available"
94+
rclone version
95+
else
96+
echo "✗ rclone command not found after module load"
97+
exit 1
98+
fi
8199
else
82-
# rclone not installed locally or in cache
83-
echo "rclone not found locally or in cache"
84-
echo "installed=false" >> $GITHUB_OUTPUT
100+
echo "✗ Failed to load rclone module"
101+
exit 1
85102
fi
86103
87-
- name: Download rclone and populate cache (no sudo)
88-
if: steps.check_rclone.outputs.installed != 'true'
89-
shell: bash
90-
run: |
91-
set -euo pipefail
92-
mkdir -p "${RCLONE_CACHE_DIR}"
93-
echo "Downloading rclone..."
94-
curl -sSL https://downloads.rclone.org/rclone-current-linux-amd64.zip -o rclone.zip
95-
unzip -q rclone.zip
96-
# Extracted directory has a dynamic name like rclone-*-linux-amd64
97-
SRC_DIR="$(echo rclone-*-linux-amd64)"
98-
mv "${SRC_DIR}/rclone" ./rclone
99-
chmod +x ./rclone
100-
# Save to cache for reuse in future runs
101-
cp ./rclone "${RCLONE_CACHE_BIN}"
102-
echo "rclone downloaded and cached at ${RCLONE_CACHE_BIN}"
103-
104104
- name: Configure rclone (local config)
105105
shell: bash
106106
run: |
@@ -123,7 +123,7 @@ runs:
123123
run: |
124124
set -euo pipefail
125125
echo "Verifying rclone configuration..."
126-
./rclone config show
126+
rclone config show
127127
128128
- name: Download archive from S3
129129
id: download
@@ -146,7 +146,7 @@ runs:
146146
echo "Archive: $archive_file"
147147
148148
# Download from S3
149-
if ./rclone copy pawsey0012:"$bucket/$archive_file" ./ --progress; then
149+
if rclone copy pawsey0012:"$bucket/$archive_file" ./ --progress; then
150150
if [ -f "./$archive_file" ]; then
151151
file_size=$(ls -lh "./$archive_file" | awk '{print $5}')
152152
echo "✓ Archive downloaded successfully from S3"
@@ -166,7 +166,7 @@ runs:
166166
echo "✗ Failed to download archive from S3"
167167
echo " Tried: s3://$bucket/$archive_file"
168168
echo "Available files in bucket:"
169-
./rclone lsf pawsey0012:"$bucket/" | head -10 || echo " (Cannot list bucket contents)"
169+
rclone lsf pawsey0012:"$bucket/" | head -10 || echo " (Cannot list bucket contents)"
170170
exit 1
171171
fi
172172
@@ -192,3 +192,92 @@ runs:
192192
echo "✗ Failed to load image: ${dockerfile_name}:${version}"
193193
exit 1
194194
fi
195+
196+
- name: Upload file to S3 storage
197+
id: upload
198+
if: inputs.upload_mode == 'true'
199+
shell: bash
200+
run: |
201+
set -euo pipefail
202+
203+
# Validate required inputs for upload mode
204+
if [ -z "${{ inputs.upload_file }}" ]; then
205+
echo "Error: upload_file is required when upload_mode=true"
206+
exit 1
207+
fi
208+
209+
upload_file="${{ inputs.upload_file }}"
210+
bucket="${{ inputs.bucket }}"
211+
file_type="${{ inputs.upload_file_type }}"
212+
213+
# Verify file exists
214+
if [ ! -f "$upload_file" ]; then
215+
echo "Error: Upload file not found: $upload_file"
216+
exit 1
217+
fi
218+
219+
echo "Uploading file to S3..."
220+
echo "File: $upload_file"
221+
echo "Bucket: $bucket"
222+
echo "Type: $file_type"
223+
224+
# Calculate file size for optimization
225+
FILE_SIZE=$(wc -c < "$upload_file")
226+
echo "File size: $FILE_SIZE bytes"
227+
228+
# Set rclone parameters based on file size
229+
if [ "$FILE_SIZE" -lt $((1024 * 1024 * 500)) ]; then
230+
# < 500MB
231+
S3_CHUNK_SIZE="16M"
232+
S3_UPLOAD_CONCURRENCY=4
233+
MULTI_THREAD_STREAMS=2
234+
elif [ "$FILE_SIZE" -lt $((1024 * 1024 * 5000)) ]; then
235+
# 500MB - 5GB
236+
S3_CHUNK_SIZE="64M"
237+
S3_UPLOAD_CONCURRENCY=8
238+
MULTI_THREAD_STREAMS=4
239+
else
240+
# > 5GB
241+
S3_CHUNK_SIZE="128M"
242+
S3_UPLOAD_CONCURRENCY=16
243+
MULTI_THREAD_STREAMS=8
244+
fi
245+
246+
echo "Optimized settings for $file_type upload:"
247+
echo " S3 chunk size: $S3_CHUNK_SIZE"
248+
echo " Upload concurrency: $S3_UPLOAD_CONCURRENCY"
249+
echo " Multi-thread streams: $MULTI_THREAD_STREAMS"
250+
251+
# Upload to S3
252+
if rclone copy "$upload_file" pawsey0012:"$bucket/" \
253+
--multi-thread-streams=$MULTI_THREAD_STREAMS \
254+
--s3-chunk-size=$S3_CHUNK_SIZE \
255+
--s3-upload-concurrency=$S3_UPLOAD_CONCURRENCY \
256+
--progress; then
257+
258+
# Get filename for verification
259+
filename=$(basename "$upload_file")
260+
261+
# Verify upload
262+
if rclone lsf pawsey0012:"$bucket/" | grep -q "^$filename$"; then
263+
file_size_human=$(ls -lh "$upload_file" | awk '{print $5}')
264+
echo "✓ File successfully uploaded to S3"
265+
echo " [BUCKET]: $bucket"
266+
echo " [S3 PATH]: $filename"
267+
echo " [SIZE]: $file_size_human"
268+
269+
# Set outputs
270+
echo "uploaded=true" >> "$GITHUB_OUTPUT"
271+
echo "s3_bucket=$bucket" >> "$GITHUB_OUTPUT"
272+
echo "s3_path=$filename" >> "$GITHUB_OUTPUT"
273+
echo "file_size=$file_size_human" >> "$GITHUB_OUTPUT"
274+
else
275+
echo "✗ Failed to verify S3 upload"
276+
exit 1
277+
fi
278+
else
279+
echo "✗ Failed to upload file to S3"
280+
echo "Available files in bucket:"
281+
rclone lsf pawsey0012:"$bucket/" | head -10 || echo " (Cannot list bucket contents)"
282+
exit 1
283+
fi

0 commit comments

Comments
 (0)