diff --git a/cicd/jenkins/Jenkinsfile b/cicd/jenkins/Jenkinsfile index 1093f6da42..e613f85727 100644 --- a/cicd/jenkins/Jenkinsfile +++ b/cicd/jenkins/Jenkinsfile @@ -163,7 +163,9 @@ pipeline { } } } - stage ('Backstop tests') { + + // --- PARALLEL TESTING STAGE --- + stage('Testing') { when { beforeAgent true allOf { @@ -172,43 +174,86 @@ pipeline { not { tag pattern: DEPLOY_TAG_PATTERN, comparator: "REGEXP" } } } - agent { - docker { - // backstop needs node 20 - image DOCKER_NODE20 + // Allows successful parallel branches to save their state even if one fails + failFast false + parallel { + + // 1. BACKSTOP: RESPONSIVE + stage('Backstop: Responsive') { + agent { docker { image DOCKER_NODE20 } } + steps { + script { + sh "export THEMES_TO_TEST='${THEMES_TO_TEST}'" + jslNpmWrapper('run tests:visual:run -- responsive', GITHUB_PUBLISH_TOKEN_CREDENTIALS) + + THEMES_TO_TEST.split(',').each { theme -> + def ciReportPath = "reports/${theme}/ci_report/responsive" + def htmlReportPath = "reports/${theme}/html_report/responsive" + + stash name: "${theme}_responsive_ci", includes: "${ciReportPath}/**" + archiveArtifacts artifacts: "${htmlReportPath}/**", fingerprint: true, allowEmptyArchive: true + jslPublishTestResults("${ciReportPath}/xunit.xml") + } + } + } } - } - steps { - script { - sh "export THEMES_TO_TEST='${THEMES_TO_TEST}'" - - jslNpmWrapper('run tests:visual:run', GITHUB_PUBLISH_TOKEN_CREDENTIALS) - stashAndArchiveCiReports() - } - } - } - stage ('E2E tests') { - when { - beforeAgent true - allOf { - not { tag pattern: RC_TAG_PATTERN, comparator: "REGEXP" } - not { tag pattern: PROMOTE_TAG_PATTERN, comparator: "REGEXP" } - not { tag pattern: DEPLOY_TAG_PATTERN, comparator: "REGEXP" } + // 2. BACKSTOP: NON-RESPONSIVE + stage('Backstop: Non-Responsive') { + agent { docker { image DOCKER_NODE20 } } + steps { + script { + sh "export THEMES_TO_TEST='${THEMES_TO_TEST}'" + jslNpmWrapper('run tests:visual:run -- non-responsive', GITHUB_PUBLISH_TOKEN_CREDENTIALS) + + THEMES_TO_TEST.split(',').each { theme -> + def ciReportPath = "reports/${theme}/ci_report/non_responsive" + def htmlReportPath = "reports/${theme}/html_report/non_responsive" + + stash name: "${theme}_non_responsive_ci", includes: "${ciReportPath}/**" + archiveArtifacts artifacts: "${htmlReportPath}/**", fingerprint: true, allowEmptyArchive: true + jslPublishTestResults("${ciReportPath}/xunit.xml") + } + } + } } - } - agent { - docker { - image DOCKER_NODE22 + + // 3. BACKSTOP: NON-RESPONSIVE CE + stage('Backstop: Non-Responsive CE') { + agent { docker { image DOCKER_NODE20 } } + steps { + script { + sh "export THEMES_TO_TEST='${THEMES_TO_TEST}'" + jslNpmWrapper('run tests:visual:run -- non-responsive-ce', GITHUB_PUBLISH_TOKEN_CREDENTIALS) + + THEMES_TO_TEST.split(',').each { theme -> + def ciReportPath = "reports/${theme}/ci_report/non_responsive_ce" + def htmlReportPath = "reports/${theme}/html_report/non_responsive_ce" + + stash name: "${theme}_non_responsive_ce_ci", includes: "${ciReportPath}/**" + archiveArtifacts artifacts: "${htmlReportPath}/**", fingerprint: true, allowEmptyArchive: true + jslPublishTestResults("${ciReportPath}/xunit.xml") + } + } + } } - } - steps { - script { - jslNpmWrapper('ci', GITHUB_PUBLISH_TOKEN_CREDENTIALS) - jslNpmWrapper('run tests:e2e', GITHUB_PUBLISH_TOKEN_CREDENTIALS) + + // 4. E2E TESTS + stage ('E2E tests') { + agent { docker { image DOCKER_NODE22 } } + steps { + script { + jslNpmWrapper('ci', GITHUB_PUBLISH_TOKEN_CREDENTIALS) + // We leave retry(2) here just in case Cypress has network timeouts + retry(2) { jslNpmWrapper('run tests:e2e', GITHUB_PUBLISH_TOKEN_CREDENTIALS) } + } + } } } } + // --- END PARALLEL TESTING STAGE --- + + // --- SONARQUBE STAGE (Runs sequentially after Testing completes) --- stage('Sonarqube') { when { beforeAgent true @@ -220,28 +265,22 @@ pipeline { } parallel { stage('Coverage') { - agent { - docker { - image DOCKER_NODE20 - } - } + agent { docker { image DOCKER_NODE20 } } steps { // TODO: Improve to have nyc, coverage (folder) & lcov.info jslQualityGateCodeCoverage(SONARQUBE_PROPERTIES) } } stage('Static Analysis') { - agent { - docker { - image DOCKER_NODE20 - } - } + agent { docker { image DOCKER_NODE20 } } steps { jslSonarQubeStaticAnalysis(SONARQUBE_PROPERTIES, SONARQUBE_CREDENTIALS) } } } } + // --- END SONARQUBE STAGE --- + stage('QualityGate') { when { beforeAgent true @@ -567,7 +606,7 @@ pipeline { } } -// This function is called to print the summary of the pipeline +// Support Functions Below def printSummary() { env.GITHUB_ORGANIZATION = jslGitGetRepoOwner() @@ -586,7 +625,6 @@ def printSummary() { """ } -// This function is called to send the build info to JIRA def sendBuildInfo() { echo "/dist & /dist/components" @@ -598,26 +636,6 @@ def sendBuildInfo() { jslJiraSendBuildInfo() } -// This function is called to stash and archive the CI reports -def stashAndArchiveCiReports() { - def tests = ["non_responsive", "responsive", "non_responsive_ce"] - - tests.each { test -> - THEMES_TO_TEST.split(',').each { theme -> - def reportBasePath = "reports/${theme}/ci_report/" - def htmlBasePath = "reports/${theme}/html_report/" - - stash name: "${theme}_${test}_ci", includes: "${reportBasePath}${test}/**" - - archiveArtifacts artifacts: "${htmlBasePath}${test}/**", fingerprint: true - jslPublishTestResults("${reportBasePath}${test}/xunit.xml") - } - } -} - -// This function is called when the pipeline fails -// - It will update the GCR as cancelled / backed out -// - It will update the DEVOPS_GCR_PREPROD_BRANCH with the PROD_BRANCH def gcrPostFailure() { def gcrInProgress = env.TAG_NAME == null ? env.GCR_NUMBER : env.TAG_NAME @@ -628,7 +646,6 @@ def gcrPostFailure() { } } -// This function is called to promote the RC tag to the DEPLOYMENT tag def promoteDeployTagGCR() { def tagAttributes = jslGCRGetTagAttributes() @@ -642,7 +659,6 @@ def promoteDeployTagGCR() { jslTriggerRemoteJob(tagName.trim(), "/${pathJob}/".toString(), false, false, 6) } -// This function is called to check the package.json version def packageVersionCheck() { sh(script: """ chmod +x ${VERSION_CHECK_SCRIPT} @@ -652,7 +668,6 @@ def packageVersionCheck() { stash name: "chi_version_bump", includes: "chi_version_bump" } -// This function is called to check if the version bump exists def isVersionBump() { def isVersionBumpResult = false @@ -682,7 +697,6 @@ def isVersionBump() { return isVersionBumpResult } -// This function is called to create a GitHub release def gitHubRelease() { def latestRelease = 'true' @@ -723,7 +737,6 @@ def gitHubRelease() { } } -// This function is called to create and open a pull request def createAndOpenPullRequest(def version) { def org = jslGitGetRepoOwner() def commitMessage = "[FSTEAM-4189] This PR is for CHI ${version}" @@ -753,7 +766,6 @@ def createAndOpenPullRequest(def version) { jslGitHubPRCreate(commitMessage, description, branchName, 'master', GITHUB_PUBLISH_TOKEN_CREDENTIALS, 'ux-chi-AssetsServer') } -// This function is called to check/create the smoke tests def smokeTestChecker() { def version = env.VERSION @@ -793,4 +805,4 @@ def smokeTestChecker() { echo "Smoke Test hook without release" jslGenerateManualTestReport("Change without Release", "Hook test", "hookTest", true, 1.0, "cypress/reports/smokeTests/smoke_test_result.xml") } -} +} \ No newline at end of file diff --git a/scripts/tests/retryFailedBackstop.js b/scripts/tests/retryFailedBackstop.js new file mode 100644 index 0000000000..3c32d053e9 --- /dev/null +++ b/scripts/tests/retryFailedBackstop.js @@ -0,0 +1,94 @@ +// File: scripts/tests/retryFailedBackstop.js +import fs from 'fs'; +import { execSync } from 'child_process'; + +// 1. Get arguments passed from the bash script +const theme = process.argv[2]; +const configName = process.argv[3]; + +if (!theme || !configName) { + console.error(`[Smart Retry] Missing arguments. Usage: node retryFailedBackstop.js `); + process.exit(1); +} + +// 2. Reconstruct the config file and report paths +const backstopConfigFile = `backstop-${configName}_${theme}.json`; + +// The bash script replaces hyphens with underscores for the report folder (e.g., non-responsive -> non_responsive) +const reportFolder = configName.replace(/-/g, '_'); +const htmlReportPath = `./reports/${theme}/html_report/${reportFolder}`; +const jsonConfigPath = `${htmlReportPath}/config.js`; + +if (!fs.existsSync(jsonConfigPath)) { + console.error(`[Smart Retry] No report found at ${jsonConfigPath}. Cannot perform targeted retry.`); + process.exit(1); +} + +// 3. Extract and parse the test results +// Backstop writes the report as a JS function wrapper: report({ ... }); +// We strip the function wrapper so we can parse it as pure JSON. +const configJsContent = fs.readFileSync(jsonConfigPath, 'utf8'); +const jsonString = configJsContent.replace(/^report\(/, '').replace(/\);?$/, ''); +let reportData; + +try { + reportData = JSON.parse(jsonString); +} catch (err) { + console.error(`[Smart Retry] Failed to parse Backstop JSON report: ${err.message}`); + process.exit(1); +} + +// 4. Identify the broken components +const failedTests = reportData.tests.filter((t) => t.status === 'fail'); + +if (failedTests.length === 0) { + console.log(`[Smart Retry] No failed tests found in the report for ${theme} - ${configName}.`); + process.exit(0); +} + +console.log(`[Smart Retry] Found ${failedTests.length} flaky test(s). Preparing targeted retry...`); + +// 5. Build the regex filter for the Backstop command +// Safely extract the label whether Backstop put it in 'pair.label' or 'meta.label' +const escapeRegExp = (string) => string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +const filterRegex = failedTests + .map((t) => { + const label = (t.pair && t.pair.label) || (t.meta && t.meta.label) || 'UnknownTest'; + return `^${escapeRegExp(label)}$`; + }) + .join('|'); + +// Construct the targeted Backstop command +const command = `npx backstop test --config=${backstopConfigFile} --filter="${filterRegex}"`; + +// 6. Execute the Retry Loop +const MAX_RETRIES = 2; // Strict but fair limit to avoid infinite hangs on real bugs +let attempt = 1; +let testsPassed = false; + +while (attempt <= MAX_RETRIES) { + console.log(`\n[Smart Retry] --- Attempt ${attempt} of ${MAX_RETRIES} ---`); + console.log(`[Smart Retry] Executing: ${command}`); + + try { + // Run the targeted backstop test. + // stdio: 'inherit' streams the colored Backstop logs directly into the Jenkins console UI. + execSync(command, { stdio: 'inherit' }); + + console.log(`\n[Smart Retry] SUCCESS: All targeted flaky tests passed on attempt ${attempt}!`); + testsPassed = true; + break; // They passed! Break out of the while loop immediately. + } catch (error) { + console.error(`\n[Smart Retry] WARNING: Targeted tests still failing on attempt ${attempt}.`); + attempt++; + } +} + +// 7. Final Verdict +if (!testsPassed) { + console.error(`\n[Smart Retry] FATAL: Tests are still failing after ${MAX_RETRIES} targeted retries.`); + console.error(`[Smart Retry] This is highly likely a legitimate visual regression, not a system glitch.`); + process.exit(1); // Tell Jenkins to turn the stage red +} else { + process.exit(0); // Tell Jenkins to turn the stage green +} diff --git a/scripts/tests/visualTests.sh b/scripts/tests/visualTests.sh index 943c1dfae4..3e4add4516 100755 --- a/scripts/tests/visualTests.sh +++ b/scripts/tests/visualTests.sh @@ -1,24 +1,28 @@ #!/bin/bash source "$(dirname "$0")/backstopConfig.sh" +# 1. Read the target config from the command line argument +TARGET_CONFIG=$1 + +if [ -z "$TARGET_CONFIG" ]; then + echo "Error: No config provided. Usage: ./visualTests.sh responsive" + exit 1 +fi + SECONDS=0 -CONFIG_FILES=("responsive" "non-responsive" "non-responsive-ce") -# Clean reports folder if [ -d "reports" ]; then rm -rf "reports" fi -echo "[CHI]: Installing dependencies..." +echo "[CHI]: Installing dependencies for $TARGET_CONFIG..." -# backstop runs on node 20 not 22, npm ci will give conflicts with package-lock # TO BE REMOVED ONCE MIGRATION TO NODE22 IS COMPLETE sed -i.bak 's/"@centurylink\/chi-documentation":[[:space:]]*"[^"]*"/"@centurylink\/chi-documentation": "1.57.0"/' package.json && rm package.json.bak mv package-lock-tests.json package-lock.json npm i -# npm ci npx playwright install npm run build @@ -26,39 +30,55 @@ npm run build npm run start:dist & SERVER_PID=$! - -# Function to perform visual tests +# Function to perform visual tests with Smart Retry test_theme () { - CONFIG=$1 + CONFIG_FILE=$1 + THEME_NAME=$2 + CONFIG_TYPE=$3 + + # Initial run + node ./scripts/tests/visualTests.js "$CONFIG_FILE" + TEST_EXIT_CODE=$? - node ./scripts/tests/visualTests.js $CONFIG - return $? + # If it failed, trigger the smart retry script + if [ $TEST_EXIT_CODE -ne 0 ]; then + echo "[CHI]: Initial run failed for $CONFIG_TYPE. Triggering Smart Retry..." + + node ./scripts/tests/retryFailedBackstop.js "$THEME_NAME" "$CONFIG_TYPE" + RETRY_EXIT_CODE=$? + + return $RETRY_EXIT_CODE + fi + + return 0 } set_backstop_config message='' -for theme in ${THEMES_TO_TEST//,/ }; -do - for config in "${CONFIG_FILES[@]}"; do - test_theme backstop-"$config"_"$theme".json +# 2. Loop only through themes, but strictly execute the TARGET_CONFIG +for theme in ${THEMES_TO_TEST//,/ }; do + echo "[CHI]: Running $TARGET_CONFIG for $theme..." + + # Pass the config file, the theme name, and the config type + test_theme "backstop-${TARGET_CONFIG}_${theme}.json" "$theme" "$TARGET_CONFIG" + + FINAL_EXIT_CODE=$? + + if [ $FINAL_EXIT_CODE -ne 0 ]; then + # Note: ensure we replace the dash with underscore for the HTML report path if that matches your folder structure + message+=$'\n'"[CHI]: FAILED TESTS: $USER_PATH/reports/$theme/html_report/${TARGET_CONFIG//-/_}/index.html" - TEST_EXIT_CODE=$? - - if [ $TEST_EXIT_CODE -ne 0 ]; then - message+=$'\n'"[CHI]: FAILED TESTS: $USER_PATH/reports/$theme/html_report/${config//-/_}/index.html" - - if [ $STOP_TESTS_ON_FAILURE -ne 0 ]; then - break 2 - fi + if [ $STOP_TESTS_ON_FAILURE -ne 0 ]; then + break fi - done + fi done minutes=$((SECONDS / 60)) seconds=$((SECONDS % 60)) -echo "[CHI]: Visual tests finished in ${minutes} minutes and ${seconds} seconds" +echo "[CHI]: Visual tests ($TARGET_CONFIG) finished in ${minutes} minutes and ${seconds} seconds" echo "$message" kill $SERVER_PID @@ -71,5 +91,4 @@ if [ -n "$message" ]; then exit 1 else exit 0 -fi - +fi \ No newline at end of file