[DPEDE-4955] TESTING IMPROVEMENTS IN JEKNINS [DONT MERGE]

cicd/jenkins/Jenkinsfile

@@ -163,7 +163,9 @@ pipeline {
         }
       }
     }
-    stage ('Backstop tests') {
+
+    // --- PARALLEL TESTING STAGE ---
+    stage('Testing') {
       when {
         beforeAgent true
         allOf {
@@ -172,43 +174,86 @@ pipeline {
           not { tag pattern: DEPLOY_TAG_PATTERN, comparator: "REGEXP" }
         }
       }
-      agent {
-        docker {
-          // backstop needs node 20
-          image DOCKER_NODE20
+      // Allows successful parallel branches to save their state even if one fails
+      failFast false 
+      parallel {
+
+        // 1. BACKSTOP: RESPONSIVE
+        stage('Backstop: Responsive') {
+          agent { docker { image DOCKER_NODE20 } }
+          steps {
+            script {
+              sh "export THEMES_TO_TEST='${THEMES_TO_TEST}'"
+              jslNpmWrapper('run tests:visual:run -- responsive', GITHUB_PUBLISH_TOKEN_CREDENTIALS)
+
+              THEMES_TO_TEST.split(',').each { theme ->
+                def ciReportPath = "reports/${theme}/ci_report/responsive"
+                def htmlReportPath = "reports/${theme}/html_report/responsive"
+
+                stash name: "${theme}_responsive_ci", includes: "${ciReportPath}/**"
+                archiveArtifacts artifacts: "${htmlReportPath}/**", fingerprint: true, allowEmptyArchive: true
+                jslPublishTestResults("${ciReportPath}/xunit.xml")
+              }
+            }
+          }
         }
-      }
-      steps {
-        script {
-          sh "export THEMES_TO_TEST='${THEMES_TO_TEST}'"
-
-          jslNpmWrapper('run tests:visual:run', GITHUB_PUBLISH_TOKEN_CREDENTIALS)
 
-          stashAndArchiveCiReports()
-        }
-      }
-    }
-    stage ('E2E tests') {
-      when {
-        beforeAgent true
-        allOf {
-          not { tag pattern: RC_TAG_PATTERN, comparator: "REGEXP" }
-          not { tag pattern: PROMOTE_TAG_PATTERN, comparator: "REGEXP" }
-          not { tag pattern: DEPLOY_TAG_PATTERN, comparator: "REGEXP" }
+        // 2. BACKSTOP: NON-RESPONSIVE
+        stage('Backstop: Non-Responsive') {
+          agent { docker { image DOCKER_NODE20 } }
+          steps {
+            script {
+              sh "export THEMES_TO_TEST='${THEMES_TO_TEST}'"
+              jslNpmWrapper('run tests:visual:run -- non-responsive', GITHUB_PUBLISH_TOKEN_CREDENTIALS)
+
+              THEMES_TO_TEST.split(',').each { theme ->
+                def ciReportPath = "reports/${theme}/ci_report/non_responsive"
+                def htmlReportPath = "reports/${theme}/html_report/non_responsive"
+
+                stash name: "${theme}_non_responsive_ci", includes: "${ciReportPath}/**"
+                archiveArtifacts artifacts: "${htmlReportPath}/**", fingerprint: true, allowEmptyArchive: true
+                jslPublishTestResults("${ciReportPath}/xunit.xml")
+              }
+            }
+          }
         }
-      }
-      agent {
-        docker {
-          image DOCKER_NODE22
+
+        // 3. BACKSTOP: NON-RESPONSIVE CE
+        stage('Backstop: Non-Responsive CE') {
+          agent { docker { image DOCKER_NODE20 } }
+          steps {
+            script {
+              sh "export THEMES_TO_TEST='${THEMES_TO_TEST}'"
+              jslNpmWrapper('run tests:visual:run -- non-responsive-ce', GITHUB_PUBLISH_TOKEN_CREDENTIALS)
+
+              THEMES_TO_TEST.split(',').each { theme ->
+                def ciReportPath = "reports/${theme}/ci_report/non_responsive_ce"
+                def htmlReportPath = "reports/${theme}/html_report/non_responsive_ce"
+
+                stash name: "${theme}_non_responsive_ce_ci", includes: "${ciReportPath}/**"
+                archiveArtifacts artifacts: "${htmlReportPath}/**", fingerprint: true, allowEmptyArchive: true
+                jslPublishTestResults("${ciReportPath}/xunit.xml")
+              }
+            }
+          }
         }
-      }
-      steps {
-        script {
-          jslNpmWrapper('ci', GITHUB_PUBLISH_TOKEN_CREDENTIALS)
-          jslNpmWrapper('run tests:e2e', GITHUB_PUBLISH_TOKEN_CREDENTIALS)
+
+        // 4. E2E TESTS
+        stage ('E2E tests') {
+          agent { docker { image DOCKER_NODE22 } }
+          steps {
+            script {
+              jslNpmWrapper('ci', GITHUB_PUBLISH_TOKEN_CREDENTIALS)
+              // We leave retry(2) here just in case Cypress has network timeouts
+              retry(2) { jslNpmWrapper('run tests:e2e', GITHUB_PUBLISH_TOKEN_CREDENTIALS) }
+            }
+          }
         }
       }
     }
+    // --- END PARALLEL TESTING STAGE ---
+
+    // --- SONARQUBE STAGE (Runs sequentially after Testing completes) ---
     stage('Sonarqube') {
       when {
         beforeAgent true
@@ -220,28 +265,22 @@ pipeline {
       }
       parallel {
         stage('Coverage') {
-          agent {
-            docker {
-              image DOCKER_NODE20
-            }
-          }
+          agent { docker { image DOCKER_NODE20 } }
           steps {
             // TODO: Improve to have nyc, coverage (folder) & lcov.info
             jslQualityGateCodeCoverage(SONARQUBE_PROPERTIES)
           }
         }
         stage('Static Analysis') {
-          agent {
-            docker {
-              image DOCKER_NODE20
-            }
-          }
+          agent { docker { image DOCKER_NODE20 } }
           steps {
             jslSonarQubeStaticAnalysis(SONARQUBE_PROPERTIES, SONARQUBE_CREDENTIALS)
           }
         }
       }
     }
+    // --- END SONARQUBE STAGE ---
+
     stage('QualityGate') {
       when {
         beforeAgent true
@@ -567,7 +606,7 @@ pipeline {
   }
 }
 
-// This function is called to print the summary of the pipeline
+// Support Functions Below
 def printSummary() {
   env.GITHUB_ORGANIZATION = jslGitGetRepoOwner()
 
@@ -586,7 +625,6 @@ def printSummary() {
   """
 }
 
-// This function is called to send the build info to JIRA
 def sendBuildInfo() {
   echo "/dist & /dist/components"
 
@@ -598,26 +636,6 @@ def sendBuildInfo() {
   jslJiraSendBuildInfo()
 }
 
-// This function is called to stash and archive the CI reports
-def stashAndArchiveCiReports() {
-  def tests = ["non_responsive", "responsive", "non_responsive_ce"]
-
-  tests.each { test ->
-    THEMES_TO_TEST.split(',').each { theme ->
-      def reportBasePath = "reports/${theme}/ci_report/"
-      def htmlBasePath = "reports/${theme}/html_report/"
-
-      stash name: "${theme}_${test}_ci", includes: "${reportBasePath}${test}/**"
-
-      archiveArtifacts artifacts: "${htmlBasePath}${test}/**", fingerprint: true
-      jslPublishTestResults("${reportBasePath}${test}/xunit.xml")
-    }
-  }
-}
-
-// This function is called when the pipeline fails
-// - It will update the GCR as cancelled / backed out
-// - It will update the DEVOPS_GCR_PREPROD_BRANCH with the PROD_BRANCH
 def gcrPostFailure() {
   def gcrInProgress = env.TAG_NAME == null ? env.GCR_NUMBER : env.TAG_NAME
 
@@ -628,7 +646,6 @@ def gcrPostFailure() {
   }
 }
 
-// This function is called to promote the RC tag to the DEPLOYMENT tag
 def promoteDeployTagGCR() {
   def tagAttributes = jslGCRGetTagAttributes()
 
@@ -642,7 +659,6 @@ def promoteDeployTagGCR() {
   jslTriggerRemoteJob(tagName.trim(), "/${pathJob}/".toString(), false, false, 6)
 }
 
-// This function is called to check the package.json version
 def packageVersionCheck() {
   sh(script: """
     chmod +x ${VERSION_CHECK_SCRIPT}
@@ -652,7 +668,6 @@ def packageVersionCheck() {
   stash name: "chi_version_bump", includes: "chi_version_bump"
 }
 
-// This function is called to check if the version bump exists
 def isVersionBump() {
   def isVersionBumpResult = false
 
@@ -682,7 +697,6 @@ def isVersionBump() {
   return isVersionBumpResult
 }
 
-// This function is called to create a GitHub release
 def gitHubRelease() {
   def latestRelease = 'true'
 
@@ -723,7 +737,6 @@ def gitHubRelease() {
   }
 }
 
-// This function is called to create and open a pull request
 def createAndOpenPullRequest(def version) {
   def org = jslGitGetRepoOwner()
   def commitMessage = "[FSTEAM-4189] This PR is for CHI ${version}"
@@ -753,7 +766,6 @@ def createAndOpenPullRequest(def version) {
   jslGitHubPRCreate(commitMessage, description, branchName, 'master', GITHUB_PUBLISH_TOKEN_CREDENTIALS, 'ux-chi-AssetsServer')
 }
 
-// This function is called to check/create the smoke tests
 def smokeTestChecker() {
   def version = env.VERSION
 
@@ -793,4 +805,4 @@ def smokeTestChecker() {
     echo "Smoke Test hook without release"
     jslGenerateManualTestReport("Change without Release", "Hook test", "hookTest", true, 1.0, "cypress/reports/smokeTests/smoke_test_result.xml")
   }
-}
+}

scripts/tests/retryFailedBackstop.js

@@ -0,0 +1,94 @@
+// File: scripts/tests/retryFailedBackstop.js
+import fs from 'fs';
+import { execSync } from 'child_process';
+
+// 1. Get arguments passed from the bash script
+const theme = process.argv[2];
+const configName = process.argv[3];
+
+if (!theme || !configName) {
+  console.error(`[Smart Retry] Missing arguments. Usage: node retryFailedBackstop.js <theme> <configName>`);
+  process.exit(1);
+}
+
+// 2. Reconstruct the config file and report paths
+const backstopConfigFile = `backstop-${configName}_${theme}.json`;
+
+// The bash script replaces hyphens with underscores for the report folder (e.g., non-responsive -> non_responsive)
+const reportFolder = configName.replace(/-/g, '_');
+const htmlReportPath = `./reports/${theme}/html_report/${reportFolder}`;
+const jsonConfigPath = `${htmlReportPath}/config.js`;
+
+if (!fs.existsSync(jsonConfigPath)) {
+  console.error(`[Smart Retry] No report found at ${jsonConfigPath}. Cannot perform targeted retry.`);
+  process.exit(1);
+}
+
+// 3. Extract and parse the test results
+// Backstop writes the report as a JS function wrapper: report({ ... });
+// We strip the function wrapper so we can parse it as pure JSON.
+const configJsContent = fs.readFileSync(jsonConfigPath, 'utf8');
+const jsonString = configJsContent.replace(/^report\(/, '').replace(/\);?$/, '');
+let reportData;
+
+try {
+  reportData = JSON.parse(jsonString);
+} catch (err) {
+  console.error(`[Smart Retry] Failed to parse Backstop JSON report: ${err.message}`);
+  process.exit(1);
+}
+
+// 4. Identify the broken components
+const failedTests = reportData.tests.filter((t) => t.status === 'fail');
+
+if (failedTests.length === 0) {
+  console.log(`[Smart Retry] No failed tests found in the report for ${theme} - ${configName}.`);
+  process.exit(0);
+}
+
+console.log(`[Smart Retry] Found ${failedTests.length} flaky test(s). Preparing targeted retry...`);
+
+// 5. Build the regex filter for the Backstop command
+// Safely extract the label whether Backstop put it in 'pair.label' or 'meta.label'
+const escapeRegExp = (string) => string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+const filterRegex = failedTests
+  .map((t) => {
+    const label = (t.pair && t.pair.label) || (t.meta && t.meta.label) || 'UnknownTest';
+    return `^${escapeRegExp(label)}$`;
+  })
+  .join('|');
+
+// Construct the targeted Backstop command
+const command = `npx backstop test --config=${backstopConfigFile} --filter="${filterRegex}"`;
+
+// 6. Execute the Retry Loop
+const MAX_RETRIES = 2; // Strict but fair limit to avoid infinite hangs on real bugs
+let attempt = 1;
+let testsPassed = false;
+
+while (attempt <= MAX_RETRIES) {
+  console.log(`\n[Smart Retry] --- Attempt ${attempt} of ${MAX_RETRIES} ---`);
+  console.log(`[Smart Retry] Executing: ${command}`);
+
+  try {
+    // Run the targeted backstop test.
+    // stdio: 'inherit' streams the colored Backstop logs directly into the Jenkins console UI.
+    execSync(command, { stdio: 'inherit' });
+
+    console.log(`\n[Smart Retry] SUCCESS: All targeted flaky tests passed on attempt ${attempt}!`);
+    testsPassed = true;
+    break; // They passed! Break out of the while loop immediately.
+  } catch (error) {
+    console.error(`\n[Smart Retry] WARNING: Targeted tests still failing on attempt ${attempt}.`);
+    attempt++;
+  }
+}
+
+// 7. Final Verdict
+if (!testsPassed) {
+  console.error(`\n[Smart Retry] FATAL: Tests are still failing after ${MAX_RETRIES} targeted retries.`);
+  console.error(`[Smart Retry] This is highly likely a legitimate visual regression, not a system glitch.`);
+  process.exit(1); // Tell Jenkins to turn the stage red
+} else {
+  process.exit(0); // Tell Jenkins to turn the stage green
+}