From 89c1cc159a32c364527db783e68a12cd8a244fc3 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Sat, 13 Dec 2025 01:48:12 -0500 Subject: [PATCH 01/25] DAOS-18348 build: Support running Bullseye Use groovy script arguments to enable Bullseye code coverage. Signed-off-by: Phil Henderson --- vars/unitTest.groovy | 25 ++++++++++++++----------- vars/unitTestPost.groovy | 5 ++++- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index 0f6c2d8a3..45892e244 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -19,6 +19,8 @@ * Or the default name has to be changed in a way that is compatible * with a future Matrix implementation. * + * config['code_coverage'] Bullseye code coverage is enabled. + * * config['coverage_stash'] Name to stash coverage artifacts * Name is based on the environment variables * for the stage if this is coverage test. @@ -71,6 +73,7 @@ * default is false. * * config['unstash_tests'] Un-stash -tests, default is true. + * */ Map afterTest(Map config, Map testRunInfo) { @@ -129,8 +132,9 @@ Map call(Map config = [:]) { String test_script = config.get('test_script', 'ci/unit/test_main.sh') Map stage_info = parseStageInfo(config) String inst_rpms = config.get('inst_rpms', '') + Boolean code_coverage = config.get('code_coverage', false) - if (stage_info['compiler'] == 'covc') { + if (code_coverage) { if (stage_info['java_pkg']) { inst_rpms += " ${stage_info['java_pkg']}" } @@ -164,13 +168,12 @@ Map call(Map config = [:]) { } } - if (stage_info['compiler'] == 'covc') { - String tools_url = env.JENKINS_URL + - 'job/daos-stack/job/tools/job/master' + - '/lastSuccessfulBuild/artifact/' + if (code_coverage) { + String tools_url = env.JENKINS_URL + + 'job/daos-stack/job/tools/job/master/lastSuccessfulBuild/artifact/' httpRequest url: tools_url + 'bullseyecoverage-linux.tar', - httpMode: 'GET', - outputFile: 'bullseye.tar' + httpMode: 'GET', + outputFile: 'bullseye.tar' } String with_valgrind = stage_info.get('with_valgrind', '') @@ -205,9 +208,9 @@ Map call(Map config = [:]) { runTestData = afterTest(p, runData) runTestData.each { resultKey, data -> runData[resultKey] = data } - if (stage_info['compiler'] == 'covc') { + if (code_coverage) { stash name: config.get('coverage_stash', "${target_stash}-unit-cov"), - includes: 'test.cov' + includes: 'test.cov' } int runTime = durationSeconds(startDate) runData['unittest_time'] = runTime @@ -223,8 +226,8 @@ Map call(Map config = [:]) { includes: results_map // Stash any optional test coverage reports for the stage - String code_coverage = 'code_coverage_' + sanitizedStageName() - stash name: code_coverage, + String code_coverage_name = 'code_coverage_' + sanitizedStageName() + stash name: code_coverage_name, includes: '**/code_coverage.json', allowEmpty: true diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index aa081e008..addfa50a0 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -10,6 +10,8 @@ * config['artifacts'] Artifacts to archive. * Default ['run_test.sh/*'] * + * config['code_coverage'] Bullseye code coverage is enabled. + * * config['referenceJobName'] Reference job name. * Defaults to 'daos-stack/daos/master' * @@ -29,6 +31,7 @@ void call(Map config = [:]) { Map stage_info = parseStageInfo(config) String cbcResult = currentBuild.currentResult + Boolean code_coverage = config.get('code_coverage', false) // Stash the Valgrind files for later analysis String valgrind_pattern = stage_info.get('valgrind_pattern', @@ -102,7 +105,7 @@ void call(Map config = [:]) { target_stash += '-' + stage_info['build_type'] } // Coverage instrumented tests and Valgrind are probably mutually exclusive - if (stage_info['compiler'] == 'covc') { + if (code_coverage) { return } From eb13e95c3a2d1aaf4f67cec4fa46738144d53b20 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Mon, 15 Dec 2025 11:18:15 -0500 Subject: [PATCH 02/25] Get bullseye code from RPM. Signed-off-by: Phil Henderson --- vars/unitTest.groovy | 7 ------- 1 file changed, 7 deletions(-) diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index 45892e244..600d84f70 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -167,13 +167,6 @@ Map call(Map config = [:]) { stashes.add("${target_stash}-install") } } - - if (code_coverage) { - String tools_url = env.JENKINS_URL + - 'job/daos-stack/job/tools/job/master/lastSuccessfulBuild/artifact/' - httpRequest url: tools_url + 'bullseyecoverage-linux.tar', - httpMode: 'GET', - outputFile: 'bullseye.tar' } String with_valgrind = stage_info.get('with_valgrind', '') From 23193b8285d99430b67b6279773965be8bfd7682 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 16 Dec 2025 09:52:41 -0500 Subject: [PATCH 03/25] Fix typo. Signed-off-by: Phil Henderson --- vars/unitTest.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index 600d84f70..e93e5afce 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -167,7 +167,6 @@ Map call(Map config = [:]) { stashes.add("${target_stash}-install") } } - } String with_valgrind = stage_info.get('with_valgrind', '') Map p = [:] From 7d64219f7ab15ad673658c8c9f74c6a559c862ad Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 17 Dec 2025 15:09:03 -0500 Subject: [PATCH 04/25] Allow empty match for bullseye stash Signed-off-by: Phil Henderson --- vars/unitTest.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index e93e5afce..24a6df0a3 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -202,7 +202,8 @@ Map call(Map config = [:]) { if (code_coverage) { stash name: config.get('coverage_stash', "${target_stash}-unit-cov"), - includes: 'test.cov' + includes: '**/test.cov' + allowEmpty: true } int runTime = durationSeconds(startDate) runData['unittest_time'] = runTime From b3fd4697ead800aaebd6cd4b3b52d784a596807a Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Sun, 21 Dec 2025 11:57:18 -0500 Subject: [PATCH 05/25] Updates. Signed-off-by: Phil Henderson --- vars/getFunctionalTestStage.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index 47055feef..d782c1cab 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -38,6 +38,7 @@ Map call(Map kwargs = [:]) { String distro = kwargs.get('distro') String image_version = kwargs.get('image_version', null) String base_branch = kwargs.get('base_branch') + String other_daos_packages = kwargs.get('other_daos_packages', 'tests-internal') String other_packages = kwargs.get('other_packages', '') Boolean run_if_pr = kwargs.get('run_if_pr', false) Boolean run_if_landing = kwargs.get('run_if_landing', false) @@ -82,7 +83,7 @@ Map call(Map kwargs = [:]) { functionalTest( image_version: image_version, inst_repos: daosRepos(distro), - inst_rpms: functionalPackages(1, next_version, 'tests-internal') + ' ' + other_packages, + inst_rpms: functionalPackages(1, next_version, other_daos_packages) + ' ' + other_packages, test_tag: tags, ftest_arg: getFunctionalArgs( pragma_suffix: pragma_suffix, From 9047a6e127ae55de1c3a5569d5dc1b9b2001ab42 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 13 Jan 2026 18:05:43 -0500 Subject: [PATCH 06/25] Support overriding parseStageInfo params for unitTests Signed-off-by: Phil Henderson --- vars/unitTest.groovy | 92 +++++++++++++++++++--------------------- vars/unitTestPost.groovy | 24 +++++++---- 2 files changed, 59 insertions(+), 57 deletions(-) diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index 24a6df0a3..640b66243 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -19,8 +19,6 @@ * Or the default name has to be changed in a way that is compatible * with a future Matrix implementation. * - * config['code_coverage'] Bullseye code coverage is enabled. - * * config['coverage_stash'] Name to stash coverage artifacts * Name is based on the environment variables * for the stage if this is coverage test. @@ -95,7 +93,7 @@ Map afterTest(Map config, Map testRunInfo) { } else { result['result'] = checkJunitFiles(testResults: testResults) } - if (config['with_valgrind'] || config['NLT']) { + if (config['with_valgrind']) { vgrcs = sh label: 'Check for Valgrind errors', script: "grep -E ')' ${valgrind_pattern} || true", returnStdout: true @@ -130,27 +128,33 @@ Map call(Map config = [:]) { long startDate = System.currentTimeMillis() String nodelist = config.get('NODELIST', env.NODELIST) String test_script = config.get('test_script', 'ci/unit/test_main.sh') - Map stage_info = parseStageInfo(config) String inst_rpms = config.get('inst_rpms', '') - Boolean code_coverage = config.get('code_coverage', false) - if (code_coverage) { - if (stage_info['java_pkg']) { - inst_rpms += " ${stage_info['java_pkg']}" - } - } + // Support backwards compatibility with parseStageInfo when config keys are ommitted + Map stage_info = parseStageInfo(config) + Integer node_count = config.get('node_count', stage_info['node_count']) + String target = config.get('target', stage_info['ci_target']) + String distro_version = config.get('distro_version', stage_info['distro_version']) + String compiler = config.get('compiler', stage_info['compiler']) + String build_type = config.get('build_type', stage_info['build_type']) + String with_valgrind = config.get('with_valgrind', '') + String always_script = config.get( + 'always_script', stage_info.get('always_script', 'ci/unit/test_post_always.sh')) + String valgrind_pattern = config.get( + 'valgrind_pattern', stage_info.get('valgrind_pattern', 'unit-test-*memcheck.xml')) + String test_results = config.get( + 'test_results', stage_info.get('testResults', 'test_results/*.xml')) Map runData = provisionNodes( - NODELIST: nodelist, - node_count: stage_info['node_count'], - distro: (stage_info['ci_target'] =~ - /([a-z]+)(.*)/)[0][1] + stage_info['distro_version'], + NODELIST: nodelist, + node_count: node_count, + distro: (target =~ /([a-z]+)(.*)/)[0][1] + distro_version, inst_repos: config.get('inst_repos', ''), inst_rpms: inst_rpms) - String target_stash = "${stage_info['target']}-${stage_info['compiler']}" - if (stage_info['build_type']) { - target_stash += '-' + stage_info['build_type'] + String target_stash = "${target}-${compiler}" + if (build_type) { + target_stash += "-${build_type}" } List stashes = [] @@ -168,43 +172,39 @@ Map call(Map config = [:]) { } } - String with_valgrind = stage_info.get('with_valgrind', '') - Map p = [:] - p['stashes'] = stashes - p['script'] = "SSH_KEY_ARGS=${env.SSH_KEY_ARGS} " + - "NODELIST=${nodelist} " + - "WITH_VALGRIND=${with_valgrind} " + - test_script - p['junit_files'] = config.get('junit_files', 'test_results/*.xml') - p['context'] = config.get('context', 'test/' + env.STAGE_NAME) - p['description'] = config.get('description', env.STAGE_NAME) + Map params = [:] + params['stashes'] = stashes + params['script'] = "SSH_KEY_ARGS=${env.SSH_KEY_ARGS} " + + "NODELIST=${nodelist} " + + "WITH_VALGRIND=${with_valgrind} " + + test_script + params['junit_files'] = config.get('junit_files', 'test_results/*.xml') + params['context'] = config.get('context', 'test/' + env.STAGE_NAME) + params['description'] = config.get('description', env.STAGE_NAME) // Do not let runTest abort the pipeline as want artifact/log collection. - p['ignore_failure'] = true + params['ignore_failure'] = true // runTest no longer knows now to notify for Unit Tests - p['notify_result'] = false + params['notify_result'] = false int time = config.get('timeout_time', 120) as int String unit = config.get('timeout_unit', 'MINUTES') Map runTestData = [:] timeout(time: time, unit: unit) { - runTestData = runTest p + runTestData = runTest params runTestData.each { resultKey, data -> runData[resultKey] = data } } - p['always_script'] = stage_info.get('always_script', - 'ci/unit/test_post_always.sh') - p['valgrind_pattern'] = stage_info.get('valgrind_pattern', - 'unit-test-*memcheck.xml') - p['testResults'] = stage_info.get('testResults', 'test_results/*.xml') - p['with_valgrind'] = with_valgrind - p['NLT'] = stage_info['NLT'] - runTestData = afterTest(p, runData) + params['always_script'] = always_script + params['valgrind_pattern'] = valgrind_pattern + params['testResults'] = test_results + params['with_valgrind'] = with_valgrind + runTestData = afterTest(params, runData) runTestData.each { resultKey, data -> runData[resultKey] = data } - if (code_coverage) { - stash name: config.get('coverage_stash', "${target_stash}-unit-cov"), - includes: '**/test.cov' - allowEmpty: true - } + // Stash the bullseye code coverage report if it was generated + stash name: config.get('coverage_stash', "${target_stash}-unit-cov"), + includes: '**/test.cov' + allowEmpty: true + int runTime = durationSeconds(startDate) runData['unittest_time'] = runTime @@ -218,11 +218,5 @@ Map call(Map config = [:]) { stash name: results_map, includes: results_map - // Stash any optional test coverage reports for the stage - String code_coverage_name = 'code_coverage_' + sanitizedStageName() - stash name: code_coverage_name, - includes: '**/code_coverage.json', - allowEmpty: true - return runData } diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index addfa50a0..70b86b314 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -33,9 +33,18 @@ void call(Map config = [:]) { String cbcResult = currentBuild.currentResult Boolean code_coverage = config.get('code_coverage', false) + // Support backwards compatibility with parseStageInfo when config keys are ommitted + String target = config.get('target', stage_info['ci_target']) + String compiler = config.get('compiler', stage_info['compiler']) + String build_type = config.get('build_type', stage_info['build_type']) + String with_valgrind = config.get('with_valgrind', '') + String valgrind_pattern = config.get( + 'valgrind_pattern', stage_info.get('valgrind_pattern', 'unit-test-*memcheck.xml')) + String testResults = config.get( + 'test_results', stage_info.get('testResults', 'test_results/*.xml')) + Boolean NLT = config.get('NLT', stage_info.get('NLT', false)) + // Stash the Valgrind files for later analysis - String valgrind_pattern = stage_info.get('valgrind_pattern', - 'unit-test-*memcheck.xml') if (config['valgrind_stash']) { try { stash name: config['valgrind_stash'], includes: valgrind_pattern @@ -59,7 +68,6 @@ void call(Map config = [:]) { List artifact_list = config.get('artifacts', ['run_test.sh/*']) - String testResults = stage_info.get('testResults', 'test_results/*.xml') if (testResults != 'None' ) { // groovylint-disable-next-line NoDouble double health_scale = 1.0 @@ -69,7 +77,7 @@ void call(Map config = [:]) { junit testResults: testResults, healthScaleFactor: health_scale } - if (stage_info['with_valgrind'] || stage_info['NLT']) { + if (with_valgrind) { String suite = sanitizedStageName() int vgfail = 0 String testdata @@ -100,16 +108,16 @@ void call(Map config = [:]) { archiveArtifacts artifacts: artifactPat, allowEmptyArchive: results['ignore_failure'] } - String target_stash = "${stage_info['target']}-${stage_info['compiler']}" - if (stage_info['build_type']) { - target_stash += '-' + stage_info['build_type'] + String target_stash = "${target}-${compiler}" + if (build_type) { + target_stash += "-${build_type}" } // Coverage instrumented tests and Valgrind are probably mutually exclusive if (code_coverage) { return } - if (stage_info['NLT']) { + if (NLT) { String cb_result = currentBuild.result discoverGitReferenceBuild(referenceJob: config.get('referenceJobName', 'daos-stack/daos/master'), From e46473f3e71ec36e8af2476047e099739f54027b Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 14 Jan 2026 16:39:31 -0500 Subject: [PATCH 07/25] Improvements. Signed-off-by: Phil Henderson --- vars/unitTest.groovy | 24 +++++++++++++----------- vars/unitTestPost.groovy | 12 +++++------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index 640b66243..b89dd5734 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -71,7 +71,6 @@ * default is false. * * config['unstash_tests'] Un-stash -tests, default is true. - * */ Map afterTest(Map config, Map testRunInfo) { @@ -93,7 +92,7 @@ Map afterTest(Map config, Map testRunInfo) { } else { result['result'] = checkJunitFiles(testResults: testResults) } - if (config['with_valgrind']) { + if (config['check_valgrind_errors']) { vgrcs = sh label: 'Check for Valgrind errors', script: "grep -E ')' ${valgrind_pattern} || true", returnStdout: true @@ -137,7 +136,8 @@ Map call(Map config = [:]) { String distro_version = config.get('distro_version', stage_info['distro_version']) String compiler = config.get('compiler', stage_info['compiler']) String build_type = config.get('build_type', stage_info['build_type']) - String with_valgrind = config.get('with_valgrind', '') + String with_valgrind = config.get('with_valgrind', stage_info.get('with_valgrind', '')) + Boolean NLT = config.get('NLT', stage_info.get('NLT', false)) String always_script = config.get( 'always_script', stage_info.get('always_script', 'ci/unit/test_post_always.sh')) String valgrind_pattern = config.get( @@ -148,9 +148,9 @@ Map call(Map config = [:]) { Map runData = provisionNodes( NODELIST: nodelist, node_count: node_count, - distro: (target =~ /([a-z]+)(.*)/)[0][1] + distro_version, - inst_repos: config.get('inst_repos', ''), - inst_rpms: inst_rpms) + distro: (target =~ /([a-z]+)(.*)/)[0][1] + distro_version, + inst_repos: config.get('inst_repos', ''), + inst_rpms: inst_rpms) String target_stash = "${target}-${compiler}" if (build_type) { @@ -196,14 +196,16 @@ Map call(Map config = [:]) { params['always_script'] = always_script params['valgrind_pattern'] = valgrind_pattern params['testResults'] = test_results - params['with_valgrind'] = with_valgrind + params['check_valgrind_errors'] = (with_valgrind || NLT) && (compiler != 'covc') runTestData = afterTest(params, runData) runTestData.each { resultKey, data -> runData[resultKey] = data } - // Stash the bullseye code coverage report if it was generated - stash name: config.get('coverage_stash', "${target_stash}-unit-cov"), - includes: '**/test.cov' - allowEmpty: true + if (compiler == 'covc') { + // Stash the bullseye code coverage report if it was generated + stash name: config.get('coverage_stash', "${target_stash}-unit-cov"), + includes: '**/test.cov' + allowEmpty: true + } int runTime = durationSeconds(startDate) runData['unittest_time'] = runTime diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index 70b86b314..331233d68 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -10,8 +10,6 @@ * config['artifacts'] Artifacts to archive. * Default ['run_test.sh/*'] * - * config['code_coverage'] Bullseye code coverage is enabled. - * * config['referenceJobName'] Reference job name. * Defaults to 'daos-stack/daos/master' * @@ -31,18 +29,18 @@ void call(Map config = [:]) { Map stage_info = parseStageInfo(config) String cbcResult = currentBuild.currentResult - Boolean code_coverage = config.get('code_coverage', false) // Support backwards compatibility with parseStageInfo when config keys are ommitted String target = config.get('target', stage_info['ci_target']) String compiler = config.get('compiler', stage_info['compiler']) String build_type = config.get('build_type', stage_info['build_type']) - String with_valgrind = config.get('with_valgrind', '') + String with_valgrind = config.get('with_valgrind', stage_info.get('with_valgrind', '')) String valgrind_pattern = config.get( 'valgrind_pattern', stage_info.get('valgrind_pattern', 'unit-test-*memcheck.xml')) String testResults = config.get( - 'test_results', stage_info.get('testResults', 'test_results/*.xml')) + 'testResults', stage_info.get('testResults', 'test_results/*.xml')) Boolean NLT = config.get('NLT', stage_info.get('NLT', false)) + Boolean check_valgrind_errors = (with_valgrind || NLT) && (compiler != 'covc') // Stash the Valgrind files for later analysis if (config['valgrind_stash']) { @@ -77,7 +75,7 @@ void call(Map config = [:]) { junit testResults: testResults, healthScaleFactor: health_scale } - if (with_valgrind) { + if (check_valgrind_errors) { String suite = sanitizedStageName() int vgfail = 0 String testdata @@ -113,7 +111,7 @@ void call(Map config = [:]) { target_stash += "-${build_type}" } // Coverage instrumented tests and Valgrind are probably mutually exclusive - if (code_coverage) { + if (compiler == 'covc') { return } From b69b8c8cea1fe38028a670e6898df6e310541c79 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Fri, 13 Feb 2026 17:42:07 -0500 Subject: [PATCH 08/25] Pass inst_rpms to functionalTest() Signed-off-by: Phil Henderson --- vars/getFunctionalTestStage.groovy | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index 80329c660..7d993b028 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -21,6 +21,9 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * distro functional test stage distro (VM) * image_version image version to use for provisioning, e.g. el8.8, leap15.6, etc. * base_branch if specified, checkout sources from this branch before running tests + * other_packages space-separated string of additional RPM packages to install + * inst_rpms space-separated string of RPM packages to install on the test nodes; + * exclusive of next_version and other_packages. * run_if_pr whether or not the stage should run for PR builds * run_if_landing whether or not the stage should run for landing builds * job_status Map of status for each stage in the job/build @@ -30,7 +33,7 @@ Map call(Map kwargs = [:]) { String name = kwargs.get('name', 'Unknown Functional Test Stage') String pragma_suffix = kwargs.get('pragma_suffix') String label = kwargs.get('label') - String next_version = kwargs.get('next_version', null) + String next_version = kwargs.get('next_version', null) // Not used if inst_rpms specified String stage_tags = kwargs.get('stage_tags') String default_tags = kwargs.get('default_tags') String nvme = kwargs.get('nvme') @@ -39,8 +42,9 @@ Map call(Map kwargs = [:]) { String distro = kwargs.get('distro') String image_version = kwargs.get('image_version', null) String base_branch = kwargs.get('base_branch') - String other_daos_packages = kwargs.get('other_daos_packages', 'tests-internal') - String other_packages = kwargs.get('other_packages', '') + String other_packages = kwargs.get('other_packages', '') // Not used if inst_rpms specified + String instRpms = kwargs.get( + 'inst_rpms', functionalPackages(1, next_version, 'tests-internal') + ' ' + other_packages) Boolean run_if_pr = kwargs.get('run_if_pr', false) Boolean run_if_landing = kwargs.get('run_if_landing', false) Map job_status = kwargs.get('job_status', [:]) @@ -84,7 +88,7 @@ Map call(Map kwargs = [:]) { functionalTest( image_version: image_version, inst_repos: daosRepos(distro), - inst_rpms: functionalPackages(1, next_version, other_daos_packages) + ' ' + other_packages, + inst_rpms: instRpms, test_tag: tags, ftest_arg: getFunctionalArgs( pragma_suffix: pragma_suffix, From de414d59060c97773b8263636145c53cfbd07fcd Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 26 Feb 2026 12:16:07 -0500 Subject: [PATCH 09/25] Adding getFunctionalPackages Signed-off-by: Phil Henderson --- vars/getFunctionalPackages.groovy | 49 ++++++++++++++++++++++++++++++ vars/getFunctionalTestStage.groovy | 8 +++-- 2 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 vars/getFunctionalPackages.groovy diff --git a/vars/getFunctionalPackages.groovy b/vars/getFunctionalPackages.groovy new file mode 100644 index 000000000..3e6633b17 --- /dev/null +++ b/vars/getFunctionalPackages.groovy @@ -0,0 +1,49 @@ +// vars/getFunctionalPackages.groovy + +/** + * getFunctionalPackages.groovy + * + * Get the packages to install in the functional test satge. + * + * @param distro functional test stage distro + * @param nextVersion next daos package version + * @param addDaosPackages additional daos-* version packages to install + * @param versionExt optional daos RPM version extension + * @param otherPackages space-separated string of additional non-daos packages to install + * @return a scripted stage to run in a pipeline + */ + +String call(String nextVersion, String daosPackages) { + String distro = parseStageInfo()['target'] + return getFunctionalPackages(distro, nextVersion, addDaosPackages, null, null) +} + +String call(String nextVersion, String daosPackages, String otherPackages) { + String distro = parseStageInfo()['target'] + return getFunctionalPackages(distro, nextVersion, addDaosPackages, otherPackages, null) +} + +String cal(String distro, String nextVersion, String daosPackages, String otherPackages, + String versionExt) { + String version = daosPackagesVersion(distro, nextVersion) + String packages = daosPackages + + // Add the build-specific version to the daos packages + if (version) { + if (distro.startsWith('ubuntu20')) { + packages += "=${version}" + } else { + packages += "-${version}" + } + if (versionExt) { + packages += versionExt + } + } + + // Add non-daos packages + if (otherPackages) { + packages += " ${otherPackages}" + } + + return packages +} diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index 7d993b028..3a4dff595 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -33,7 +33,6 @@ Map call(Map kwargs = [:]) { String name = kwargs.get('name', 'Unknown Functional Test Stage') String pragma_suffix = kwargs.get('pragma_suffix') String label = kwargs.get('label') - String next_version = kwargs.get('next_version', null) // Not used if inst_rpms specified String stage_tags = kwargs.get('stage_tags') String default_tags = kwargs.get('default_tags') String nvme = kwargs.get('nvme') @@ -42,9 +41,12 @@ Map call(Map kwargs = [:]) { String distro = kwargs.get('distro') String image_version = kwargs.get('image_version', null) String base_branch = kwargs.get('base_branch') - String other_packages = kwargs.get('other_packages', '') // Not used if inst_rpms specified String instRpms = kwargs.get( - 'inst_rpms', functionalPackages(1, next_version, 'tests-internal') + ' ' + other_packages) + 'inst_rpms', + getFunctionalPackages( + kwargs.get('next_version', null), + 'daos{,-{client,tests,server,serialize,tests-internal}', + kwargs.get('other_packages', null))) Boolean run_if_pr = kwargs.get('run_if_pr', false) Boolean run_if_landing = kwargs.get('run_if_landing', false) Map job_status = kwargs.get('job_status', [:]) From 48b168b9a1090d370ce23dd1731485b4816e3126 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 26 Feb 2026 16:34:59 -0500 Subject: [PATCH 10/25] Fix typo Signed-off-by: Phil Henderson --- vars/getFunctionalPackages.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/getFunctionalPackages.groovy b/vars/getFunctionalPackages.groovy index 3e6633b17..b2e58ebbc 100644 --- a/vars/getFunctionalPackages.groovy +++ b/vars/getFunctionalPackages.groovy @@ -15,12 +15,12 @@ String call(String nextVersion, String daosPackages) { String distro = parseStageInfo()['target'] - return getFunctionalPackages(distro, nextVersion, addDaosPackages, null, null) + return getFunctionalPackages(distro, nextVersion, daosPackages, null, null) } String call(String nextVersion, String daosPackages, String otherPackages) { String distro = parseStageInfo()['target'] - return getFunctionalPackages(distro, nextVersion, addDaosPackages, otherPackages, null) + return getFunctionalPackages(distro, nextVersion, daosPackages, otherPackages, null) } String cal(String distro, String nextVersion, String daosPackages, String otherPackages, From f5fae432023df1a7e4a7ad8a7da5dc0be9cdb42f Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 26 Feb 2026 16:46:54 -0500 Subject: [PATCH 11/25] Adding getAdditionalPackages.groovy Signed-off-by: Phil Henderson --- vars/getAdditionalPackages.groovy | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 vars/getAdditionalPackages.groovy diff --git a/vars/getAdditionalPackages.groovy b/vars/getAdditionalPackages.groovy new file mode 100644 index 000000000..c515a2da8 --- /dev/null +++ b/vars/getAdditionalPackages.groovy @@ -0,0 +1,25 @@ +// vars/getAdditionalPackages.groovy + +/** + * + * getAdditionalPackages.groovy + * + * Get the additional packages for the functional test stages based on the provider and + * whether or not bullseye reporting is enabled. + * + * @ param ucx whether or not to include UCX packages + * @ param bullseye whether or not the packages are bullseye versioned + * @ return a String of space-separated package names + */ +String call(Boolean ucx=false, Boolean bullseye=false) { + String packages = '' + if (ucx) { + packages += ' mercury-ucx' + } else { + packages += ' mercury-libfabric' + } + if (bullseye) { + packages += ' bullseye' + } + return packages.trim() +} From 656d985d92e36ece10071632645a1df5fbcab4e1 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 26 Feb 2026 17:10:53 -0500 Subject: [PATCH 12/25] Fix null strings Signed-off-by: Phil Henderson --- vars/getFunctionalPackages.groovy | 4 ++-- vars/getFunctionalTestStage.groovy | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vars/getFunctionalPackages.groovy b/vars/getFunctionalPackages.groovy index b2e58ebbc..7605fda8e 100644 --- a/vars/getFunctionalPackages.groovy +++ b/vars/getFunctionalPackages.groovy @@ -15,12 +15,12 @@ String call(String nextVersion, String daosPackages) { String distro = parseStageInfo()['target'] - return getFunctionalPackages(distro, nextVersion, daosPackages, null, null) + return getFunctionalPackages(distro, nextVersion, daosPackages, '', '') } String call(String nextVersion, String daosPackages, String otherPackages) { String distro = parseStageInfo()['target'] - return getFunctionalPackages(distro, nextVersion, daosPackages, otherPackages, null) + return getFunctionalPackages(distro, nextVersion, daosPackages, otherPackages, '') } String cal(String distro, String nextVersion, String daosPackages, String otherPackages, diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index 3a4dff595..09e5c203a 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -44,9 +44,9 @@ Map call(Map kwargs = [:]) { String instRpms = kwargs.get( 'inst_rpms', getFunctionalPackages( - kwargs.get('next_version', null), + kwargs.get('next_version', ''), 'daos{,-{client,tests,server,serialize,tests-internal}', - kwargs.get('other_packages', null))) + kwargs.get('other_packages', ''))) Boolean run_if_pr = kwargs.get('run_if_pr', false) Boolean run_if_landing = kwargs.get('run_if_landing', false) Map job_status = kwargs.get('job_status', [:]) From 07109509a2551527f541d9705c5ecd9118fee7d5 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 26 Feb 2026 17:29:22 -0500 Subject: [PATCH 13/25] Fixed call definition Signed-off-by: Phil Henderson --- vars/getFunctionalPackages.groovy | 6 +++--- vars/getFunctionalTestStage.groovy | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/vars/getFunctionalPackages.groovy b/vars/getFunctionalPackages.groovy index 7605fda8e..c832a1468 100644 --- a/vars/getFunctionalPackages.groovy +++ b/vars/getFunctionalPackages.groovy @@ -15,15 +15,15 @@ String call(String nextVersion, String daosPackages) { String distro = parseStageInfo()['target'] - return getFunctionalPackages(distro, nextVersion, daosPackages, '', '') + return getFunctionalPackages(distro, nextVersion, daosPackages, null, null) } String call(String nextVersion, String daosPackages, String otherPackages) { String distro = parseStageInfo()['target'] - return getFunctionalPackages(distro, nextVersion, daosPackages, otherPackages, '') + return getFunctionalPackages(distro, nextVersion, daosPackages, otherPackages, null) } -String cal(String distro, String nextVersion, String daosPackages, String otherPackages, +String call(String distro, String nextVersion, String daosPackages, String otherPackages, String versionExt) { String version = daosPackagesVersion(distro, nextVersion) String packages = daosPackages diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index 09e5c203a..3a4dff595 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -44,9 +44,9 @@ Map call(Map kwargs = [:]) { String instRpms = kwargs.get( 'inst_rpms', getFunctionalPackages( - kwargs.get('next_version', ''), + kwargs.get('next_version', null), 'daos{,-{client,tests,server,serialize,tests-internal}', - kwargs.get('other_packages', ''))) + kwargs.get('other_packages', null))) Boolean run_if_pr = kwargs.get('run_if_pr', false) Boolean run_if_landing = kwargs.get('run_if_landing', false) Map job_status = kwargs.get('job_status', [:]) From e7f88eaa21ff10036288cb8d55047d1cf2f1a640 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 26 Feb 2026 18:50:13 -0500 Subject: [PATCH 14/25] Fix typo Signed-off-by: Phil Henderson --- vars/getFunctionalTestStage.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index 3a4dff595..a5861820d 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -45,7 +45,7 @@ Map call(Map kwargs = [:]) { 'inst_rpms', getFunctionalPackages( kwargs.get('next_version', null), - 'daos{,-{client,tests,server,serialize,tests-internal}', + 'daos{,-{client,tests,server,serialize,tests-internal}}', kwargs.get('other_packages', null))) Boolean run_if_pr = kwargs.get('run_if_pr', false) Boolean run_if_landing = kwargs.get('run_if_landing', false) From 5f0b9374ecd60769924e8bbb125a2f9964bd9030 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 26 Feb 2026 18:56:36 -0500 Subject: [PATCH 15/25] Updates Signed-off-by: Phil Henderson --- vars/getFunctionalPackages.groovy | 8 +++++++- vars/getFunctionalTestStage.groovy | 4 +--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/vars/getFunctionalPackages.groovy b/vars/getFunctionalPackages.groovy index c832a1468..a00334ea4 100644 --- a/vars/getFunctionalPackages.groovy +++ b/vars/getFunctionalPackages.groovy @@ -26,7 +26,13 @@ String call(String nextVersion, String daosPackages, String otherPackages) { String call(String distro, String nextVersion, String daosPackages, String otherPackages, String versionExt) { String version = daosPackagesVersion(distro, nextVersion) - String packages = daosPackages + String packages = '' + + if (daosPackages) { + packages += daosPackages + } else { + packages += 'daos{,-{client,tests,server,serialize,tests-internal}}' + } // Add the build-specific version to the daos packages if (version) { diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index a5861820d..e1b131365 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -44,9 +44,7 @@ Map call(Map kwargs = [:]) { String instRpms = kwargs.get( 'inst_rpms', getFunctionalPackages( - kwargs.get('next_version', null), - 'daos{,-{client,tests,server,serialize,tests-internal}}', - kwargs.get('other_packages', null))) + kwargs.get('next_version', null), null, kwargs.get('other_packages', null))) Boolean run_if_pr = kwargs.get('run_if_pr', false) Boolean run_if_landing = kwargs.get('run_if_landing', false) Map job_status = kwargs.get('job_status', [:]) From da1f60cfa6f9086a31040bb28396c74327b424d8 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 26 Feb 2026 18:59:57 -0500 Subject: [PATCH 16/25] Updates Signed-off-by: Phil Henderson --- vars/getFunctionalPackages.groovy | 8 ++++---- vars/getFunctionalTestStage.groovy | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/vars/getFunctionalPackages.groovy b/vars/getFunctionalPackages.groovy index a00334ea4..1cbef2945 100644 --- a/vars/getFunctionalPackages.groovy +++ b/vars/getFunctionalPackages.groovy @@ -13,14 +13,14 @@ * @return a scripted stage to run in a pipeline */ -String call(String nextVersion, String daosPackages) { +String call(String nextVersion, String otherPackages) { String distro = parseStageInfo()['target'] - return getFunctionalPackages(distro, nextVersion, daosPackages, null, null) + return getFunctionalPackages(distro, nextVersion, null, otherPackages, null) } -String call(String nextVersion, String daosPackages, String otherPackages) { +String call(String nextVersion, String otherPackages, String versionExt) { String distro = parseStageInfo()['target'] - return getFunctionalPackages(distro, nextVersion, daosPackages, otherPackages, null) + return getFunctionalPackages(distro, nextVersion, null, otherPackages, versionExt) } String call(String distro, String nextVersion, String daosPackages, String otherPackages, diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index e1b131365..a7f8ba905 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -43,8 +43,7 @@ Map call(Map kwargs = [:]) { String base_branch = kwargs.get('base_branch') String instRpms = kwargs.get( 'inst_rpms', - getFunctionalPackages( - kwargs.get('next_version', null), null, kwargs.get('other_packages', null))) + getFunctionalPackages(kwargs.get('next_version', null), kwargs.get('other_packages', null))) Boolean run_if_pr = kwargs.get('run_if_pr', false) Boolean run_if_landing = kwargs.get('run_if_landing', false) Map job_status = kwargs.get('job_status', [:]) From b1f0ff605cd8fec5e8a251894039d3d521ea8ac8 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 5 Mar 2026 14:01:05 -0500 Subject: [PATCH 17/25] Updates Signed-off-by: Phil Henderson --- vars/getFunctionalPackages.groovy | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/vars/getFunctionalPackages.groovy b/vars/getFunctionalPackages.groovy index 1cbef2945..1327e3022 100644 --- a/vars/getFunctionalPackages.groovy +++ b/vars/getFunctionalPackages.groovy @@ -7,25 +7,31 @@ * * @param distro functional test stage distro * @param nextVersion next daos package version - * @param addDaosPackages additional daos-* version packages to install - * @param versionExt optional daos RPM version extension + * @param daosPackages daos packages to install (with a version) * @param otherPackages space-separated string of additional non-daos packages to install + * @param bullseye option to include the '.bullseye' extension to the daos package version * @return a scripted stage to run in a pipeline */ String call(String nextVersion, String otherPackages) { String distro = parseStageInfo()['target'] - return getFunctionalPackages(distro, nextVersion, null, otherPackages, null) + return getFunctionalPackages(distro, nextVersion, null, otherPackages, false) } -String call(String nextVersion, String otherPackages, String versionExt) { +String call(String nextVersion, String otherPackages, Boolean bullseye=false) { String distro = parseStageInfo()['target'] - return getFunctionalPackages(distro, nextVersion, null, otherPackages, versionExt) + return getFunctionalPackages(distro, nextVersion, null, otherPackages, bullseye) +} + +String call(String nextVersion, Boolean ucx=false, Boolean bullseye=false) { + String distro = parseStageInfo()['target'] + String otherPackages = getAdditionalPackages(ucx, bullseye) + return getFunctionalPackages(distro, nextVersion, null, otherPackages, bullseye) } String call(String distro, String nextVersion, String daosPackages, String otherPackages, - String versionExt) { - String version = daosPackagesVersion(distro, nextVersion) + Boolean bullseye=false) { + String version = daosPackagesVersion(distro, nextVersion, bullseye) String packages = '' if (daosPackages) { @@ -41,8 +47,8 @@ String call(String distro, String nextVersion, String daosPackages, String other } else { packages += "-${version}" } - if (versionExt) { - packages += versionExt + if (bullseye) { + packages += '.bullseye' } } From 5d120d3886dc78c631376ad5f1c66d934301d869 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Fri, 6 Mar 2026 11:20:27 -0500 Subject: [PATCH 18/25] Update functional test test.cov stash Signed-off-by: Phil Henderson --- vars/runTestFunctionalV2.groovy | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/vars/runTestFunctionalV2.groovy b/vars/runTestFunctionalV2.groovy index 985f21b46..fc5e1d091 100644 --- a/vars/runTestFunctionalV2.groovy +++ b/vars/runTestFunctionalV2.groovy @@ -88,14 +88,16 @@ Map call(Map config = [:]) { // Restore the ignore failure setting config['ignore_failure'] = ignore_failure - String coverageFile = 'test.cov' - if (!fileExists('test.cov')) { - coverageFile += '_not_done' - fileOperations([fileCreateOperation(fileName: coverageFile, - fileContent: '')]) + // Stash code coverage file if it exists + if (fileExists("${basedir}bullseye_coverage_logs/test.cov")) { + String name = 'func' + stage_info['pragma_suffix'] + '-cov' + println("[${env.STAGE_NAME}] Stashing ${basedir}bullseye_coverage_logs/test.cov in ${name}") + dir("${basedir}bullseye_coverage_logs") { + stash name: config.get('coverage_stash', name), + includes: 'test.cov' + } + } else { + println("[${env.STAGE_NAME}] No ${basedir}bullseye_coverage_logs/test.cov file found!") } - String name = 'func' + stage_info['pragma_suffix'] + '-cov' - stash name: config.get('coverage_stash', name), - includes: coverageFile return runData } From 66ca107b0b04b38e9994726dd946a882c5cf623a Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Mon, 9 Mar 2026 09:27:36 -0400 Subject: [PATCH 19/25] Updates Signed-off-by: Phil Henderson --- vars/functionalTest.groovy | 2 ++ vars/getFunctionalPackages.groovy | 5 ----- vars/getFunctionalTestStage.groovy | 5 ++++- vars/runTestFunctionalV2.groovy | 21 +++++++++++++-------- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/vars/functionalTest.groovy b/vars/functionalTest.groovy index 2d9520200..0e605b512 100755 --- a/vars/functionalTest.groovy +++ b/vars/functionalTest.groovy @@ -66,6 +66,7 @@ Map call(Map config = [:]) { String nodelist = config.get('NODELIST', env.NODELIST) String context = config.get('context', 'test/' + env.STAGE_NAME) String description = config.get('description', env.STAGE_NAME) + String coverage_stash = config.get('coverage_stash', '') Map stage_info = parseStageInfo(config) @@ -113,6 +114,7 @@ Map call(Map config = [:]) { run_test_config['ftest_arg'] = config.get('ftest_arg', stage_info['ftest_arg']) run_test_config['context'] = context run_test_config['description'] = description + run_test_config['coverage_stash'] = coverage_stash Map runtestData = [:] if (config.get('test_function', 'runTestFunctional') == diff --git a/vars/getFunctionalPackages.groovy b/vars/getFunctionalPackages.groovy index 1327e3022..6065b0131 100644 --- a/vars/getFunctionalPackages.groovy +++ b/vars/getFunctionalPackages.groovy @@ -13,11 +13,6 @@ * @return a scripted stage to run in a pipeline */ -String call(String nextVersion, String otherPackages) { - String distro = parseStageInfo()['target'] - return getFunctionalPackages(distro, nextVersion, null, otherPackages, false) -} - String call(String nextVersion, String otherPackages, Boolean bullseye=false) { String distro = parseStageInfo()['target'] return getFunctionalPackages(distro, nextVersion, null, otherPackages, bullseye) diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index a7f8ba905..df6b7a0b6 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -27,6 +27,7 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * run_if_pr whether or not the stage should run for PR builds * run_if_landing whether or not the stage should run for landing builds * job_status Map of status for each stage in the job/build + * coverage_stash name of stash to include code coverage results from the tests * @return a scripted stage to run in a pipeline */ Map call(Map kwargs = [:]) { @@ -47,6 +48,7 @@ Map call(Map kwargs = [:]) { Boolean run_if_pr = kwargs.get('run_if_pr', false) Boolean run_if_landing = kwargs.get('run_if_landing', false) Map job_status = kwargs.get('job_status', [:]) + String coverage_stash = kwargs.get('coverage_stash', '') return { stage("${name}") { @@ -94,7 +96,8 @@ Map call(Map kwargs = [:]) { nvme: nvme, default_nvme: default_nvme, provider: provider)['ftest_arg'], - test_function: 'runTestFunctionalV2')) + test_function: 'runTestFunctionalV2', + coverage_stash: coverage_stash)) } finally { println("[${name}] Running functionalTestPostV2()") functionalTestPostV2() diff --git a/vars/runTestFunctionalV2.groovy b/vars/runTestFunctionalV2.groovy index fc5e1d091..482c34f21 100644 --- a/vars/runTestFunctionalV2.groovy +++ b/vars/runTestFunctionalV2.groovy @@ -89,15 +89,20 @@ Map call(Map config = [:]) { config['ignore_failure'] = ignore_failure // Stash code coverage file if it exists - if (fileExists("${basedir}bullseye_coverage_logs/test.cov")) { - String name = 'func' + stage_info['pragma_suffix'] + '-cov' - println("[${env.STAGE_NAME}] Stashing ${basedir}bullseye_coverage_logs/test.cov in ${name}") - dir("${basedir}bullseye_coverage_logs") { - stash name: config.get('coverage_stash', name), - includes: 'test.cov' + String coverage_stash = config.get('coverage_stash', '') + if (coverage_stash) { + if (fileExists("${basedir}bullseye_coverage_logs/test.cov")) { + // String name = 'func' + stage_info['pragma_suffix'] + '-cov' + println("[${env.STAGE_NAME}] Stashing ${basedir}bullseye_coverage_logs/test.cov in " + "${coverage_stash}") + dir("${basedir}bullseye_coverage_logs") { + stash name: config.get('coverage_stash', coverage_stash), + includes: 'test.cov' + } + } else { + println("[${env.STAGE_NAME}] No ${basedir}bullseye_coverage_logs/test.cov file found!") } - } else { - println("[${env.STAGE_NAME}] No ${basedir}bullseye_coverage_logs/test.cov file found!") } + return runData } From f29b92e8f19036fde8af823279e7e4789ef1081a Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Mon, 9 Mar 2026 19:13:19 -0400 Subject: [PATCH 20/25] Fix unintended change. Signed-off-by: Phil Henderson --- vars/getFunctionalPackages.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/getFunctionalPackages.groovy b/vars/getFunctionalPackages.groovy index 6065b0131..c0a3bd6f0 100644 --- a/vars/getFunctionalPackages.groovy +++ b/vars/getFunctionalPackages.groovy @@ -26,7 +26,7 @@ String call(String nextVersion, Boolean ucx=false, Boolean bullseye=false) { String call(String distro, String nextVersion, String daosPackages, String otherPackages, Boolean bullseye=false) { - String version = daosPackagesVersion(distro, nextVersion, bullseye) + String version = daosPackagesVersion(distro, nextVersion) String packages = '' if (daosPackages) { From 86cace022f8353648606cbf498d557d950f80837 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 10 Mar 2026 09:24:23 -0400 Subject: [PATCH 21/25] Fix functional test.cov stash Signed-off-by: Phil Henderson --- vars/runTestFunctionalV2.groovy | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/vars/runTestFunctionalV2.groovy b/vars/runTestFunctionalV2.groovy index 482c34f21..5315408b0 100644 --- a/vars/runTestFunctionalV2.groovy +++ b/vars/runTestFunctionalV2.groovy @@ -90,17 +90,17 @@ Map call(Map config = [:]) { // Stash code coverage file if it exists String coverage_stash = config.get('coverage_stash', '') + String coverage_dir = "${basedir}bullseye_coverage_logs" + String coverage_file = "${coverage_dir}/test.cov" if (coverage_stash) { - if (fileExists("${basedir}bullseye_coverage_logs/test.cov")) { + if (fileExists(coverage_file)) { // String name = 'func' + stage_info['pragma_suffix'] + '-cov' - println("[${env.STAGE_NAME}] Stashing ${basedir}bullseye_coverage_logs/test.cov in " - "${coverage_stash}") - dir("${basedir}bullseye_coverage_logs") { - stash name: config.get('coverage_stash', coverage_stash), - includes: 'test.cov' + println("[${env.STAGE_NAME}] Stashing ${coverage_file} in ${coverage_stash}") + dir(coverage_dir) { + stash name: coverage_stash, includes: 'test.cov' } } else { - println("[${env.STAGE_NAME}] No ${basedir}bullseye_coverage_logs/test.cov file found!") + println("[${env.STAGE_NAME}] No ${coverage_file} file found for ${coverage_stash}!") } } From be8d9db8404e96c2887b2abc2d0608a62680b3c3 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 10 Mar 2026 21:41:51 -0400 Subject: [PATCH 22/25] Added message Signed-off-by: Phil Henderson --- vars/getFunctionalPackages.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vars/getFunctionalPackages.groovy b/vars/getFunctionalPackages.groovy index c0a3bd6f0..3db4a870f 100644 --- a/vars/getFunctionalPackages.groovy +++ b/vars/getFunctionalPackages.groovy @@ -52,5 +52,7 @@ String call(String distro, String nextVersion, String daosPackages, String other packages += " ${otherPackages}" } + println("getFunctionalPackages: distro=${distro}, version=${version}, bullseye=${bullseye}, otherPackages=${otherPackages} => ${packages}") + return packages } From 9de1a4ebd1d2ed26666bcb2a99c7b424d53fcf4f Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 11 Mar 2026 18:25:31 -0400 Subject: [PATCH 23/25] Simplify getFunctionalPackages Signed-off-by: Phil Henderson --- vars/getFunctionalPackages.groovy | 32 ++++++++---------------------- vars/getFunctionalTestStage.groovy | 6 ++---- 2 files changed, 10 insertions(+), 28 deletions(-) diff --git a/vars/getFunctionalPackages.groovy b/vars/getFunctionalPackages.groovy index 3db4a870f..e1308d47e 100644 --- a/vars/getFunctionalPackages.groovy +++ b/vars/getFunctionalPackages.groovy @@ -5,54 +5,38 @@ * * Get the packages to install in the functional test satge. * - * @param distro functional test stage distro - * @param nextVersion next daos package version * @param daosPackages daos packages to install (with a version) * @param otherPackages space-separated string of additional non-daos packages to install * @param bullseye option to include the '.bullseye' extension to the daos package version * @return a scripted stage to run in a pipeline */ -String call(String nextVersion, String otherPackages, Boolean bullseye=false) { - String distro = parseStageInfo()['target'] - return getFunctionalPackages(distro, nextVersion, null, otherPackages, bullseye) +String call(String otherPackages, Boolean bullseye=false) { + return getFunctionalPackages(null, otherPackages, bullseye) } -String call(String nextVersion, Boolean ucx=false, Boolean bullseye=false) { - String distro = parseStageInfo()['target'] +String call(Boolean ucx=false, Boolean bullseye=false) { String otherPackages = getAdditionalPackages(ucx, bullseye) - return getFunctionalPackages(distro, nextVersion, null, otherPackages, bullseye) + return getFunctionalPackages(null, otherPackages, bullseye) } -String call(String distro, String nextVersion, String daosPackages, String otherPackages, - Boolean bullseye=false) { - String version = daosPackagesVersion(distro, nextVersion) +String call(String daosPackages, String otherPackages, Boolean bullseye=false) { String packages = '' if (daosPackages) { packages += daosPackages + } elif (bullseye) { + packages += 'daos-bullseye{,-{client,tests,server,serialize,tests-internal}}' } else { packages += 'daos{,-{client,tests,server,serialize,tests-internal}}' } - // Add the build-specific version to the daos packages - if (version) { - if (distro.startsWith('ubuntu20')) { - packages += "=${version}" - } else { - packages += "-${version}" - } - if (bullseye) { - packages += '.bullseye' - } - } - // Add non-daos packages if (otherPackages) { packages += " ${otherPackages}" } - println("getFunctionalPackages: distro=${distro}, version=${version}, bullseye=${bullseye}, otherPackages=${otherPackages} => ${packages}") + println("getFunctionalPackages(${daosPackages}, ${otherPackages}, ${bullseye}) => ${packages}") return packages } diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index df6b7a0b6..c2d0e1a16 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -12,7 +12,6 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * name functional test stage name * pragma_suffix functional test stage commit pragma suffix, e.g. '-hw-medium' * label functional test stage default cluster label - * next_version next daos package version * stage_tags functional test stage tags always used and combined with all other tags * default_tags launch.py tags argument to use when no parameter or commit pragma exist * nvme launch.py --nvme argument to use @@ -23,7 +22,7 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * base_branch if specified, checkout sources from this branch before running tests * other_packages space-separated string of additional RPM packages to install * inst_rpms space-separated string of RPM packages to install on the test nodes; - * exclusive of next_version and other_packages. + * exclusive of other_packages. * run_if_pr whether or not the stage should run for PR builds * run_if_landing whether or not the stage should run for landing builds * job_status Map of status for each stage in the job/build @@ -43,8 +42,7 @@ Map call(Map kwargs = [:]) { String image_version = kwargs.get('image_version', null) String base_branch = kwargs.get('base_branch') String instRpms = kwargs.get( - 'inst_rpms', - getFunctionalPackages(kwargs.get('next_version', null), kwargs.get('other_packages', null))) + 'inst_rpms', getFunctionalPackages(kwargs.get('other_packages', null))) Boolean run_if_pr = kwargs.get('run_if_pr', false) Boolean run_if_landing = kwargs.get('run_if_landing', false) Map job_status = kwargs.get('job_status', [:]) From 7ebaa153e76a4d383a40e6bc939b8becf1474424 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 12 Mar 2026 09:05:19 -0400 Subject: [PATCH 24/25] Fix syntax Signed-off-by: Phil Henderson --- vars/getFunctionalPackages.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/getFunctionalPackages.groovy b/vars/getFunctionalPackages.groovy index e1308d47e..aa94b20f0 100644 --- a/vars/getFunctionalPackages.groovy +++ b/vars/getFunctionalPackages.groovy @@ -25,7 +25,7 @@ String call(String daosPackages, String otherPackages, Boolean bullseye=false) { if (daosPackages) { packages += daosPackages - } elif (bullseye) { + } else if (bullseye) { packages += 'daos-bullseye{,-{client,tests,server,serialize,tests-internal}}' } else { packages += 'daos{,-{client,tests,server,serialize,tests-internal}}' From 2940e5ea9d389bc08067ab2bbd810a486418721d Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 12 Mar 2026 13:36:41 -0400 Subject: [PATCH 25/25] Fix 'Commit Pragma tests'. Now that skipping functional VM tests does not take into account whether or not the build is a PR, remove the criteria from the expected skip result. Signed-off-by: Phil Henderson --- Jenkinsfile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ce4574717..b2a655d73 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -433,7 +433,7 @@ pipeline { 'Functional Hardware Large'] commits = [[pragmas: [''], /* groovylint-disable-next-line UnnecessaryGetter */ - skips: [isPr(), isPr(), false, isPr(), !isPr(), !isPr(), true, !isPr()]], + skips: [false, false, false, false, !isPr(), !isPr(), true, !isPr()]], [pragmas: ['Skip-test: true'], /* groovylint-disable-next-line UnnecessaryGetter */ skips: [true, true, true, true, true, true, true, true]], @@ -460,37 +460,37 @@ pipeline { skips: [false, false, false, false, !isPr(), !isPr(), true, !isPr()]], [pragmas: ['Skip-func-test-hw: true'], /* groovylint-disable-next-line UnnecessaryGetter */ - skips: [isPr(), isPr(), false, isPr(), true, true, true, true]], + skips: [false, false, false, false, true, true, true, true]], [pragmas: ['Skip-func-test-hw-medium: true\n' + 'Skip-func-test-hw-medium-verbs-provider: true\n' + 'Skip-func-test-hw-medium-ucx-provider: true\n' + 'Skip-func-test-hw-large: true'], /* groovylint-disable-next-line UnnecessaryGetter */ - skips: [isPr(), isPr(), false, isPr(), true, true, true, true]], + skips: [false, false, false, false, true, true, true, true]], [pragmas: ['Skip-func-test-hw-medium: false\n' + 'Skip-func-test-hw-medium-verbs-provider: false\n' + 'Skip-func-test-hw-medium-ucx-provider: false\n' + 'Skip-func-test-hw-large: false'], /* groovylint-disable-next-line UnnecessaryGetter */ - skips: [isPr(), isPr(), false, isPr(), false, false, false, false]], + skips: [false, false, false, false, false, false, false, false]], [pragmas: ['Skip-func-hw-test: true'], /* groovylint-disable-next-line UnnecessaryGetter */ - skips: [isPr(), isPr(), false, isPr(), true, true, true, true]], + skips: [false, false, false, false, true, true, true, true]], [pragmas: ['Skip-func-hw-test-medium: true\n' + 'Skip-func-hw-test-medium-verbs-provider: true\n' + 'Skip-func-hw-test-medium-ucx-provider: true\n' + 'Skip-func-hw-test-large: true'], /* groovylint-disable-next-line UnnecessaryGetter */ - skips: [isPr(), isPr(), false, isPr(), true, true, true, true]], + skips: [false, false, false, false, true, true, true, true]], [pragmas: ['Skip-func-hw-test-medium: false\n' + 'Skip-func-hw-test-medium-verbs-provider: false\n' + 'Skip-func-hw-test-medium-ucx-provider: false\n' + 'Skip-func-hw-test-large: false'], /* groovylint-disable-next-line UnnecessaryGetter */ - skips: [isPr(), isPr(), false, isPr(), false, false, false, false]], + skips: [false, false, false, false, false, false, false, false]], [pragmas: ['Run-daily-stages: true'], /* groovylint-disable-next-line UnnecessaryGetter */ - skips: [isPr(), isPr(), false, isPr(), false, false, false, false]], + skips: [false, false, false, false, false, false, false, false]], [pragmas: ['Skip-build-leap15-rpm: true\n' + 'Skip-build-el7-rpm: true\n' + 'Skip-build-el8-rpm: true\n' + @@ -502,7 +502,7 @@ pipeline { 'Skip-build-el8-rpm: false\n' + 'Skip-build-el9-rpm: false'], /* groovylint-disable-next-line UnnecessaryGetter */ - skips: [isPr(), isPr(), false, isPr(), !isPr(), !isPr(), true, !isPr()]]] + skips: [false, false, false, false, !isPr(), !isPr(), true, !isPr()]]] errors = 0 commits.each { commit -> cm = 'Test commit\n\n'