DAOS-18348 build: Support running Bullseye

Jenkinsfile

@@ -433,7 +433,7 @@ pipeline {
                                       'Functional Hardware Large']
                             commits = [[pragmas: [''],
                                         /* groovylint-disable-next-line UnnecessaryGetter */
-                                        skips: [isPr(), isPr(), false, isPr(), !isPr(), !isPr(), true, !isPr()]],
+                                        skips: [false, false, false, false, !isPr(), !isPr(), true, !isPr()]],
                                        [pragmas: ['Skip-test: true'],
                                         /* groovylint-disable-next-line UnnecessaryGetter */
                                         skips: [true, true, true, true, true, true, true, true]],
@@ -460,37 +460,37 @@ pipeline {
                                         skips: [false, false, false, false, !isPr(), !isPr(), true, !isPr()]],
                                        [pragmas: ['Skip-func-test-hw: true'],
                                         /* groovylint-disable-next-line UnnecessaryGetter */
-                                        skips: [isPr(), isPr(), false, isPr(), true, true, true, true]],
+                                        skips: [false, false, false, false, true, true, true, true]],
                                        [pragmas: ['Skip-func-test-hw-medium: true\n' +
                                                   'Skip-func-test-hw-medium-verbs-provider: true\n' +
                                                   'Skip-func-test-hw-medium-ucx-provider: true\n' +
                                                   'Skip-func-test-hw-large: true'],
                                         /* groovylint-disable-next-line UnnecessaryGetter */
-                                        skips: [isPr(), isPr(), false, isPr(), true, true, true, true]],
+                                        skips: [false, false, false, false, true, true, true, true]],
                                        [pragmas: ['Skip-func-test-hw-medium: false\n' +
                                                   'Skip-func-test-hw-medium-verbs-provider: false\n' +
                                                   'Skip-func-test-hw-medium-ucx-provider: false\n' +
                                                   'Skip-func-test-hw-large: false'],
                                         /* groovylint-disable-next-line UnnecessaryGetter */
-                                        skips: [isPr(), isPr(), false, isPr(), false, false, false, false]],
+                                        skips: [false, false, false, false, false, false, false, false]],
                                        [pragmas: ['Skip-func-hw-test: true'],
                                         /* groovylint-disable-next-line UnnecessaryGetter */
-                                        skips: [isPr(), isPr(), false, isPr(), true, true, true, true]],
+                                        skips: [false, false, false, false, true, true, true, true]],
                                        [pragmas: ['Skip-func-hw-test-medium: true\n' +
                                                   'Skip-func-hw-test-medium-verbs-provider: true\n' +
                                                   'Skip-func-hw-test-medium-ucx-provider: true\n' +
                                                   'Skip-func-hw-test-large: true'],
                                         /* groovylint-disable-next-line UnnecessaryGetter */
-                                        skips: [isPr(), isPr(), false, isPr(), true, true, true, true]],
+                                        skips: [false, false, false, false, true, true, true, true]],
                                        [pragmas: ['Skip-func-hw-test-medium: false\n' +
                                                   'Skip-func-hw-test-medium-verbs-provider: false\n' +
                                                   'Skip-func-hw-test-medium-ucx-provider: false\n' +
                                                   'Skip-func-hw-test-large: false'],
                                         /* groovylint-disable-next-line UnnecessaryGetter */
-                                        skips: [isPr(), isPr(), false, isPr(), false, false, false, false]],
+                                        skips: [false, false, false, false, false, false, false, false]],
                                        [pragmas: ['Run-daily-stages: true'],
                                         /* groovylint-disable-next-line UnnecessaryGetter */
-                                        skips: [isPr(), isPr(), false, isPr(), false, false, false, false]],
+                                        skips: [false, false, false, false, false, false, false, false]],
                                        [pragmas: ['Skip-build-leap15-rpm: true\n' +
                                                   'Skip-build-el7-rpm: true\n' +
                                                   'Skip-build-el8-rpm: true\n' +
@@ -502,7 +502,7 @@ pipeline {
                                                   'Skip-build-el8-rpm: false\n' +
                                                   'Skip-build-el9-rpm: false'],
                                         /* groovylint-disable-next-line UnnecessaryGetter */
-                                        skips: [isPr(), isPr(), false, isPr(), !isPr(), !isPr(), true, !isPr()]]]
+                                        skips: [false, false, false, false, !isPr(), !isPr(), true, !isPr()]]]
                             errors = 0
                             commits.each { commit ->
                                 cm = 'Test commit\n\n'

vars/functionalTest.groovy

@@ -66,6 +66,7 @@ Map call(Map config = [:]) {
     String nodelist = config.get('NODELIST', env.NODELIST)
     String context = config.get('context', 'test/' + env.STAGE_NAME)
     String description = config.get('description', env.STAGE_NAME)
+    String coverage_stash = config.get('coverage_stash', '')
 
     Map stage_info = parseStageInfo(config)
 
@@ -113,6 +114,7 @@ Map call(Map config = [:]) {
     run_test_config['ftest_arg'] = config.get('ftest_arg', stage_info['ftest_arg'])
     run_test_config['context'] = context
     run_test_config['description'] = description
+    run_test_config['coverage_stash'] = coverage_stash
 
     Map runtestData = [:]
     if (config.get('test_function', 'runTestFunctional') ==

vars/getAdditionalPackages.groovy

@@ -0,0 +1,25 @@
+// vars/getAdditionalPackages.groovy
+
+/**
+ *
+ * getAdditionalPackages.groovy
+ *
+ * Get the additional packages for the functional test stages based on the provider and
+ * whether or not bullseye reporting is enabled.
+ *
+ * @ param ucx              whether or not to include UCX packages
+ * @ param bullseye         whether or not the packages are bullseye versioned
+ * @ return a String of space-separated package names
+ */
+String call(Boolean ucx=false, Boolean bullseye=false) {
+    String packages = ''
+    if (ucx) {
+        packages += ' mercury-ucx'
+    } else {
+        packages += ' mercury-libfabric'
+    }
+    if (bullseye) {
+        packages += ' bullseye'
+    }
+    return packages.trim()
+}

vars/getFunctionalPackages.groovy

@@ -0,0 +1,42 @@
+// vars/getFunctionalPackages.groovy
+
+/**
+ * getFunctionalPackages.groovy
+ *
+ * Get the packages to install in the functional test satge.
+ *
+ * @param daosPackages      daos packages to install (with a version)
+ * @param otherPackages     space-separated string of additional non-daos packages to install
+ * @param bullseye          option to include the '.bullseye' extension to the daos package version
+ * @return a scripted stage to run in a pipeline
+ */
+
+String call(String otherPackages, Boolean bullseye=false) {
+    return getFunctionalPackages(null, otherPackages, bullseye)
+}
+
+String call(Boolean ucx=false, Boolean bullseye=false) {
+    String otherPackages = getAdditionalPackages(ucx, bullseye)
+    return getFunctionalPackages(null, otherPackages, bullseye)
+}
+
+String call(String daosPackages, String otherPackages, Boolean bullseye=false) {
+    String packages = ''
+
+    if (daosPackages) {
+        packages += daosPackages
+    } else if (bullseye) {
+        packages += 'daos-bullseye{,-{client,tests,server,serialize,tests-internal}}'
+    } else {
+        packages += 'daos{,-{client,tests,server,serialize,tests-internal}}'
+    }
+
+    // Add non-daos packages
+    if (otherPackages) {
+        packages += " ${otherPackages}"
+    }
+
+    println("getFunctionalPackages(${daosPackages}, ${otherPackages}, ${bullseye}) => ${packages}")
+
+    return packages
+}

vars/getFunctionalTestStage.groovy

@@ -12,7 +12,6 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
  *      name            functional test stage name
  *      pragma_suffix   functional test stage commit pragma suffix, e.g. '-hw-medium'
  *      label           functional test stage default cluster label
- *      next_version    next daos package version
  *      stage_tags      functional test stage tags always used and combined with all other tags
  *      default_tags    launch.py tags argument to use when no parameter or commit pragma exist
  *      nvme            launch.py --nvme argument to use
@@ -21,16 +20,19 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
  *      distro          functional test stage distro (VM)
  *      image_version   image version to use for provisioning, e.g. el8.8, leap15.6, etc.
  *      base_branch     if specified, checkout sources from this branch before running tests
+ *      other_packages  space-separated string of additional RPM packages to install
+ *      inst_rpms       space-separated string of RPM packages to install on the test nodes;
+ *                          exclusive of other_packages.
  *      run_if_pr       whether or not the stage should run for PR builds
  *      run_if_landing  whether or not the stage should run for landing builds
  *      job_status      Map of status for each stage in the job/build
+ *      coverage_stash  name of stash to include code coverage results from the tests
  * @return a scripted stage to run in a pipeline
  */
 Map call(Map kwargs = [:]) {
     String name = kwargs.get('name', 'Unknown Functional Test Stage')
     String pragma_suffix = kwargs.get('pragma_suffix')
     String label = kwargs.get('label')
-    String next_version = kwargs.get('next_version', null)
     String stage_tags = kwargs.get('stage_tags')
     String default_tags = kwargs.get('default_tags')
     String nvme = kwargs.get('nvme')
@@ -39,10 +41,12 @@ Map call(Map kwargs = [:]) {
     String distro = kwargs.get('distro')
     String image_version = kwargs.get('image_version', null)
     String base_branch = kwargs.get('base_branch')
-    String other_packages = kwargs.get('other_packages', '')
+    String instRpms = kwargs.get(
+        'inst_rpms', getFunctionalPackages(kwargs.get('other_packages', null)))
     Boolean run_if_pr = kwargs.get('run_if_pr', false)
     Boolean run_if_landing = kwargs.get('run_if_landing', false)
     Map job_status = kwargs.get('job_status', [:])
+    String coverage_stash = kwargs.get('coverage_stash', '')
 
     return {
         stage("${name}") {
@@ -83,14 +87,15 @@ Map call(Map kwargs = [:]) {
                             functionalTest(
                                 image_version: image_version,
                                 inst_repos: daosRepos(distro),
-                                inst_rpms: functionalPackages(1, next_version, 'tests-internal') + ' ' + other_packages,
+                                inst_rpms: instRpms,
                                 test_tag: tags,
                                 ftest_arg: getFunctionalArgs(
                                     pragma_suffix: pragma_suffix,
                                     nvme: nvme,
                                     default_nvme: default_nvme,
                                     provider: provider)['ftest_arg'],
-                                test_function: 'runTestFunctionalV2'))
+                                test_function: 'runTestFunctionalV2',
+                                coverage_stash: coverage_stash))
                     } finally {
                         println("[${name}] Running functionalTestPostV2()")
                         functionalTestPostV2()

vars/runTestFunctionalV2.groovy

@@ -88,14 +88,21 @@ Map call(Map config = [:]) {
     // Restore the ignore failure setting
     config['ignore_failure'] = ignore_failure
 
-    String coverageFile = 'test.cov'
-    if (!fileExists('test.cov')) {
-        coverageFile += '_not_done'
-        fileOperations([fileCreateOperation(fileName: coverageFile,
-                                            fileContent: '')])
+    // Stash code coverage file if it exists
+    String coverage_stash = config.get('coverage_stash', '')
+    String coverage_dir = "${basedir}bullseye_coverage_logs"
+    String coverage_file = "${coverage_dir}/test.cov"
+    if (coverage_stash) {
+        if (fileExists(coverage_file)) {
+            // String name = 'func' + stage_info['pragma_suffix'] + '-cov'
+            println("[${env.STAGE_NAME}] Stashing ${coverage_file} in ${coverage_stash}")
+            dir(coverage_dir) {
+                stash name: coverage_stash, includes: 'test.cov'
+            }
+        } else {
+            println("[${env.STAGE_NAME}] No ${coverage_file} file found for ${coverage_stash}!")
+        }
     }
-    String name = 'func' + stage_info['pragma_suffix'] + '-cov'
-    stash name: config.get('coverage_stash', name),
-          includes: coverageFile
+
     return runData
 }

vars/unitTest.groovy

@@ -94,7 +94,7 @@ Map afterTest(Map config, Map testRunInfo) {
     } else {
         result['result'] = checkJunitFiles(testResults: testResults)
     }
-    if (config['with_valgrind'] || config['NLT']) {
+    if (config['check_valgrind_errors']) {
         vgrcs = sh label: 'Check for Valgrind errors',
                    script: "grep -E '<error( |>)' ${valgrind_pattern} || true",
                    returnStdout: true
@@ -129,31 +129,40 @@ Map call(Map config = [:]) {
     long startDate = System.currentTimeMillis()
     String nodelist = config.get('NODELIST', env.NODELIST)
     String test_script = config.get('test_script', 'ci/unit/test_main.sh')
-    Map stage_info = parseStageInfo(config)
     String inst_rpms = config.get('inst_rpms', '')
 
-    if (stage_info['compiler'] == 'covc') {
-        if (stage_info['java_pkg']) {
-            inst_rpms += " ${stage_info['java_pkg']}"
-        }
-    }
+    // Support backwards compatibility with parseStageInfo when config keys are ommitted
+    Map stage_info = parseStageInfo(config)
+    Integer node_count = config.get('node_count', stage_info['node_count'])
+    String target = config.get('target', stage_info['ci_target'])
+    String distro_version = config.get('distro_version', stage_info['distro_version'])
+    String compiler = config.get('compiler', stage_info['compiler'])
+    String build_type = config.get('build_type', stage_info['build_type'])
+    String with_valgrind = config.get('with_valgrind', stage_info.get('with_valgrind', ''))
+    Boolean NLT = config.get('NLT', stage_info.get('NLT', false))
+    String always_script = config.get(
+        'always_script', stage_info.get('always_script', 'ci/unit/test_post_always.sh'))
+    String valgrind_pattern = config.get(
+        'valgrind_pattern', stage_info.get('valgrind_pattern', 'unit-test-*memcheck.xml'))
+    String test_results = config.get(
+        'test_results', stage_info.get('testResults', 'test_results/*.xml'))
 
     String image_version = config.get('image_version', '') ?:
-        (stage_info['ci_target'] =~ /([a-z]+)(.*)/)[0][1] + stage_info['distro_version']
+        (target  =~ /([a-z]+)(.*)/)[0][1] + distro_version
 
     Map runData = provisionNodes(
                  NODELIST: nodelist,
-                 node_count: stage_info['node_count'],
+                 node_count: node_count,
                  distro: image_version,
                  inst_repos: config.get('inst_repos', ''),
                  inst_rpms: inst_rpms)
 
     /* el9-gcc-tests */
-    String target_stash = (image_version ?: ${stage_info['target']}).split('\\.')[0]
+    String target_stash = (image_version ?: target).split('\\.')[0]
 
-    target_stash += '-' + stage_info['compiler']
-    if (stage_info['build_type']) {
-        target_stash += '-' + stage_info['build_type']
+    target_stash += '-' + compiler
+    if (build_type) {
+        target_stash += '-' + build_type
     }
 
     List stashes = []
@@ -171,51 +180,41 @@ Map call(Map config = [:]) {
         }
     }
 
-    if (stage_info['compiler'] == 'covc') {
-        String tools_url = env.JENKINS_URL +
-                    'job/daos-stack/job/tools/job/master' +
-                    '/lastSuccessfulBuild/artifact/'
-        httpRequest url: tools_url + 'bullseyecoverage-linux.tar',
-                httpMode: 'GET',
-                outputFile: 'bullseye.tar'
-    }
-
-    String with_valgrind = stage_info.get('with_valgrind', '')
-    Map p = [:]
-    p['stashes'] = stashes
-    p['script'] = "SSH_KEY_ARGS=${env.SSH_KEY_ARGS} " +
-                  "NODELIST=${nodelist} " +
-                  "WITH_VALGRIND=${with_valgrind} " +
-                  test_script
-    p['junit_files'] = config.get('junit_files', 'test_results/*.xml')
-    p['context'] = config.get('context', 'test/' + env.STAGE_NAME)
-    p['description'] = config.get('description', env.STAGE_NAME)
+    Map params = [:]
+    params['stashes'] = stashes
+    params['script'] = "SSH_KEY_ARGS=${env.SSH_KEY_ARGS} " +
+                       "NODELIST=${nodelist} " +
+                       "WITH_VALGRIND=${with_valgrind} " +
+                       test_script
+    params['junit_files'] = config.get('junit_files', 'test_results/*.xml')
+    params['context'] = config.get('context', 'test/' + env.STAGE_NAME)
+    params['description'] = config.get('description', env.STAGE_NAME)
     // Do not let runTest abort the pipeline as want artifact/log collection.
-    p['ignore_failure'] = true
+    params['ignore_failure'] = true
     // runTest no longer knows now to notify for Unit Tests
-    p['notify_result'] = false
+    params['notify_result'] = false
     int time = config.get('timeout_time', 120) as int
     String unit = config.get('timeout_unit', 'MINUTES')
 
     Map runTestData = [:]
     timeout(time: time, unit: unit) {
-        runTestData = runTest p
+        runTestData = runTest params
         runTestData.each { resultKey, data -> runData[resultKey] = data }
     }
-    p['always_script'] = stage_info.get('always_script',
-                                        'ci/unit/test_post_always.sh')
-    p['valgrind_pattern'] = stage_info.get('valgrind_pattern',
-                                           'unit-test-*memcheck.xml')
-    p['testResults'] = stage_info.get('testResults', 'test_results/*.xml')
-    p['with_valgrind'] = with_valgrind
-    p['NLT'] = stage_info['NLT']
-    runTestData = afterTest(p, runData)
+    params['always_script'] = always_script
+    params['valgrind_pattern'] = valgrind_pattern
+    params['testResults'] = test_results
+    params['check_valgrind_errors'] = (with_valgrind || NLT) && (compiler != 'covc')
+    runTestData = afterTest(params, runData)
     runTestData.each { resultKey, data -> runData[resultKey] = data }
 
-    if (stage_info['compiler'] == 'covc') {
+    if (compiler == 'covc') {
+        // Stash the bullseye code coverage report if it was generated
         stash name: config.get('coverage_stash', "${target_stash}-unit-cov"),
-            includes: 'test.cov'
+              includes: '**/test.cov'
+              allowEmpty: true
     }
+
     int runTime = durationSeconds(startDate)
     runData['unittest_time'] = runTime
 
@@ -229,11 +228,5 @@ Map call(Map config = [:]) {
     stash name: results_map,
           includes: results_map
 
-    // Stash any optional test coverage reports for the stage
-    String code_coverage = 'code_coverage_' + sanitizedStageName()
-    stash name: code_coverage,
-          includes: '**/code_coverage.json',
-          allowEmpty: true
-
     return runData
 }