Add S3 Client Retry Mechanism

README.md

@@ -29,6 +29,7 @@ jobs:
           sessionToken: "AQoDYXdzEJraDcqRtz123" # optional
           bucket: actions-cache # required
           use-fallback: true # optional, use github actions cache fallback, default true
+          retry: true # optional, enable retry on failure s3 operations, default false
 
           # actions/cache compatible properties: https://github.com/actions/cache
           key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}

action.yml

@@ -44,6 +44,14 @@ inputs:
     description: "Check if a restore is successfull but dont download/extract cache."
     required: false
     default: "false"
+  retry:
+    description: "Number of retries to perform on failure"
+    required: false
+    default: "false"
+  retry-count:
+    description: "Number of times to retry the minio operation in case of failure"
+    required: false
+    default: "3"
   # zip-option:
   #   description: zip options
   #   required: false

package.json

@@ -10,7 +10,8 @@
     "@actions/cache": "^4.0.3",
     "@actions/core": "^1.11.1",
     "@types/node": "^20.5.1",
-    "minio": "^7.1.1"
+    "minio": "^7.1.1",
+    "p-retry": "^6.2.1"
   },
   "devDependencies": {
     "@types/jest": "^26.0.23",

restore/action.yml

@@ -44,6 +44,14 @@ inputs:
     description: "Check if a restore is successfull but dont download/extract cache."
     required: false
     default: "false"
+  retry:
+    description: "Number of retries to perform on failure"
+    required: false
+    default: "false"
+  retry-count:
+    description: "Number of times to retry the minio operation in case of failure"
+    required: false
+    default: "3"
   # zip-option:
   #   description: zip options
   #   required: false

save/action.yml

@@ -37,6 +37,14 @@ inputs:
     description: "Use github actions/cache as fallback"
     required: false
     default: "true"
+  retry:
+    description: "Number of retries to perform on failure"
+    required: false
+    default: "false"
+  retry-count:
+    description: "Number of times to retry the minio operation in case of failure"
+    required: false
+    default: "3"
   # zip-option:
   #   description: zip options
   #   required: false

src/restore.ts

@@ -15,6 +15,7 @@ import {
   setCacheSizeOutput,
   saveMatchedKey,
   getInput,
+  withRetry,
 } from "./utils";
 
 process.on("uncaughtException", (e) => core.info("warning: " + e.message));
@@ -80,6 +81,7 @@ async function restoreCache() {
         core.info(
           `Downloading cache from s3 to ${archivePath}. bucket: ${bucket}, object: ${obj.name}`,
         );
+        await withRetry("fGetObject", () => mc.fGetObject(bucket, obj.name, archivePath));
         await mc.fGetObject(bucket, obj.name, archivePath);
 
         if (core.isDebug()) {

src/utils.ts

@@ -6,6 +6,7 @@ import { State } from "./state";
 import path from "path";
 import {createTar, listTar} from "@actions/cache/lib/internal/tar";
 import * as cache from "@actions/cache";
+import pRetry from 'p-retry';
 
 export function isGhes(): boolean {
   const ghUrl = new URL(
@@ -47,6 +48,21 @@ export function newMinio({
   });
 }
 
+export function withRetry<A>(name: string, fn: () => Promise<A>): Promise<A> {
+  if (getInputAsBoolean("retry")) {
+    return pRetry(fn, {
+      retries: getInputAsInt("retry-count") ?? 3,
+      onFailedAttempt: (error) => {
+        core.info(
+          `Failed to ${name}. Attempt ${error.attemptNumber} failed. ${error.message}`
+        );
+      },
+    });
+  } else {
+    return fn();
+  }
+}
+
 export function getInputAsBoolean(
   name: string,
   options?: core.InputOptions
@@ -229,7 +245,7 @@ export async function saveCache(standalone: boolean) {
       const object = path.join(key, cacheFileName);
 
       core.info(`Uploading tar to s3. Bucket: ${bucket}, Object: ${object}`);
-      await mc.fPutObject(bucket, object, archivePath, {});
+      await withRetry("fPutObject", () => mc.fPutObject(bucket, object, archivePath, {}));
       core.info("Cache saved to s3 successfully");
     } catch (e) {
       if (useFallback) {