diff --git a/Jenkinsfile b/Jenkinsfile index ce4574717..b2a655d73 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -433,7 +433,7 @@ pipeline { 'Functional Hardware Large'] commits = [[pragmas: [''], /* groovylint-disable-next-line UnnecessaryGetter */ - skips: [isPr(), isPr(), false, isPr(), !isPr(), !isPr(), true, !isPr()]], + skips: [false, false, false, false, !isPr(), !isPr(), true, !isPr()]], [pragmas: ['Skip-test: true'], /* groovylint-disable-next-line UnnecessaryGetter */ skips: [true, true, true, true, true, true, true, true]], @@ -460,37 +460,37 @@ pipeline { skips: [false, false, false, false, !isPr(), !isPr(), true, !isPr()]], [pragmas: ['Skip-func-test-hw: true'], /* groovylint-disable-next-line UnnecessaryGetter */ - skips: [isPr(), isPr(), false, isPr(), true, true, true, true]], + skips: [false, false, false, false, true, true, true, true]], [pragmas: ['Skip-func-test-hw-medium: true\n' + 'Skip-func-test-hw-medium-verbs-provider: true\n' + 'Skip-func-test-hw-medium-ucx-provider: true\n' + 'Skip-func-test-hw-large: true'], /* groovylint-disable-next-line UnnecessaryGetter */ - skips: [isPr(), isPr(), false, isPr(), true, true, true, true]], + skips: [false, false, false, false, true, true, true, true]], [pragmas: ['Skip-func-test-hw-medium: false\n' + 'Skip-func-test-hw-medium-verbs-provider: false\n' + 'Skip-func-test-hw-medium-ucx-provider: false\n' + 'Skip-func-test-hw-large: false'], /* groovylint-disable-next-line UnnecessaryGetter */ - skips: [isPr(), isPr(), false, isPr(), false, false, false, false]], + skips: [false, false, false, false, false, false, false, false]], [pragmas: ['Skip-func-hw-test: true'], /* groovylint-disable-next-line UnnecessaryGetter */ - skips: [isPr(), isPr(), false, isPr(), true, true, true, true]], + skips: [false, false, false, false, true, true, true, true]], [pragmas: ['Skip-func-hw-test-medium: true\n' + 'Skip-func-hw-test-medium-verbs-provider: true\n' + 'Skip-func-hw-test-medium-ucx-provider: true\n' + 'Skip-func-hw-test-large: true'], /* groovylint-disable-next-line UnnecessaryGetter */ - skips: [isPr(), isPr(), false, isPr(), true, true, true, true]], + skips: [false, false, false, false, true, true, true, true]], [pragmas: ['Skip-func-hw-test-medium: false\n' + 'Skip-func-hw-test-medium-verbs-provider: false\n' + 'Skip-func-hw-test-medium-ucx-provider: false\n' + 'Skip-func-hw-test-large: false'], /* groovylint-disable-next-line UnnecessaryGetter */ - skips: [isPr(), isPr(), false, isPr(), false, false, false, false]], + skips: [false, false, false, false, false, false, false, false]], [pragmas: ['Run-daily-stages: true'], /* groovylint-disable-next-line UnnecessaryGetter */ - skips: [isPr(), isPr(), false, isPr(), false, false, false, false]], + skips: [false, false, false, false, false, false, false, false]], [pragmas: ['Skip-build-leap15-rpm: true\n' + 'Skip-build-el7-rpm: true\n' + 'Skip-build-el8-rpm: true\n' + @@ -502,7 +502,7 @@ pipeline { 'Skip-build-el8-rpm: false\n' + 'Skip-build-el9-rpm: false'], /* groovylint-disable-next-line UnnecessaryGetter */ - skips: [isPr(), isPr(), false, isPr(), !isPr(), !isPr(), true, !isPr()]]] + skips: [false, false, false, false, !isPr(), !isPr(), true, !isPr()]]] errors = 0 commits.each { commit -> cm = 'Test commit\n\n' diff --git a/vars/functionalTest.groovy b/vars/functionalTest.groovy index 2d9520200..0e605b512 100755 --- a/vars/functionalTest.groovy +++ b/vars/functionalTest.groovy @@ -66,6 +66,7 @@ Map call(Map config = [:]) { String nodelist = config.get('NODELIST', env.NODELIST) String context = config.get('context', 'test/' + env.STAGE_NAME) String description = config.get('description', env.STAGE_NAME) + String coverage_stash = config.get('coverage_stash', '') Map stage_info = parseStageInfo(config) @@ -113,6 +114,7 @@ Map call(Map config = [:]) { run_test_config['ftest_arg'] = config.get('ftest_arg', stage_info['ftest_arg']) run_test_config['context'] = context run_test_config['description'] = description + run_test_config['coverage_stash'] = coverage_stash Map runtestData = [:] if (config.get('test_function', 'runTestFunctional') == diff --git a/vars/getAdditionalPackages.groovy b/vars/getAdditionalPackages.groovy new file mode 100644 index 000000000..c515a2da8 --- /dev/null +++ b/vars/getAdditionalPackages.groovy @@ -0,0 +1,25 @@ +// vars/getAdditionalPackages.groovy + +/** + * + * getAdditionalPackages.groovy + * + * Get the additional packages for the functional test stages based on the provider and + * whether or not bullseye reporting is enabled. + * + * @ param ucx whether or not to include UCX packages + * @ param bullseye whether or not the packages are bullseye versioned + * @ return a String of space-separated package names + */ +String call(Boolean ucx=false, Boolean bullseye=false) { + String packages = '' + if (ucx) { + packages += ' mercury-ucx' + } else { + packages += ' mercury-libfabric' + } + if (bullseye) { + packages += ' bullseye' + } + return packages.trim() +} diff --git a/vars/getFunctionalPackages.groovy b/vars/getFunctionalPackages.groovy new file mode 100644 index 000000000..aa94b20f0 --- /dev/null +++ b/vars/getFunctionalPackages.groovy @@ -0,0 +1,42 @@ +// vars/getFunctionalPackages.groovy + +/** + * getFunctionalPackages.groovy + * + * Get the packages to install in the functional test satge. + * + * @param daosPackages daos packages to install (with a version) + * @param otherPackages space-separated string of additional non-daos packages to install + * @param bullseye option to include the '.bullseye' extension to the daos package version + * @return a scripted stage to run in a pipeline + */ + +String call(String otherPackages, Boolean bullseye=false) { + return getFunctionalPackages(null, otherPackages, bullseye) +} + +String call(Boolean ucx=false, Boolean bullseye=false) { + String otherPackages = getAdditionalPackages(ucx, bullseye) + return getFunctionalPackages(null, otherPackages, bullseye) +} + +String call(String daosPackages, String otherPackages, Boolean bullseye=false) { + String packages = '' + + if (daosPackages) { + packages += daosPackages + } else if (bullseye) { + packages += 'daos-bullseye{,-{client,tests,server,serialize,tests-internal}}' + } else { + packages += 'daos{,-{client,tests,server,serialize,tests-internal}}' + } + + // Add non-daos packages + if (otherPackages) { + packages += " ${otherPackages}" + } + + println("getFunctionalPackages(${daosPackages}, ${otherPackages}, ${bullseye}) => ${packages}") + + return packages +} diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index 45fd1ce1b..c2d0e1a16 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -12,7 +12,6 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * name functional test stage name * pragma_suffix functional test stage commit pragma suffix, e.g. '-hw-medium' * label functional test stage default cluster label - * next_version next daos package version * stage_tags functional test stage tags always used and combined with all other tags * default_tags launch.py tags argument to use when no parameter or commit pragma exist * nvme launch.py --nvme argument to use @@ -21,16 +20,19 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * distro functional test stage distro (VM) * image_version image version to use for provisioning, e.g. el8.8, leap15.6, etc. * base_branch if specified, checkout sources from this branch before running tests + * other_packages space-separated string of additional RPM packages to install + * inst_rpms space-separated string of RPM packages to install on the test nodes; + * exclusive of other_packages. * run_if_pr whether or not the stage should run for PR builds * run_if_landing whether or not the stage should run for landing builds * job_status Map of status for each stage in the job/build + * coverage_stash name of stash to include code coverage results from the tests * @return a scripted stage to run in a pipeline */ Map call(Map kwargs = [:]) { String name = kwargs.get('name', 'Unknown Functional Test Stage') String pragma_suffix = kwargs.get('pragma_suffix') String label = kwargs.get('label') - String next_version = kwargs.get('next_version', null) String stage_tags = kwargs.get('stage_tags') String default_tags = kwargs.get('default_tags') String nvme = kwargs.get('nvme') @@ -39,10 +41,12 @@ Map call(Map kwargs = [:]) { String distro = kwargs.get('distro') String image_version = kwargs.get('image_version', null) String base_branch = kwargs.get('base_branch') - String other_packages = kwargs.get('other_packages', '') + String instRpms = kwargs.get( + 'inst_rpms', getFunctionalPackages(kwargs.get('other_packages', null))) Boolean run_if_pr = kwargs.get('run_if_pr', false) Boolean run_if_landing = kwargs.get('run_if_landing', false) Map job_status = kwargs.get('job_status', [:]) + String coverage_stash = kwargs.get('coverage_stash', '') return { stage("${name}") { @@ -83,14 +87,15 @@ Map call(Map kwargs = [:]) { functionalTest( image_version: image_version, inst_repos: daosRepos(distro), - inst_rpms: functionalPackages(1, next_version, 'tests-internal') + ' ' + other_packages, + inst_rpms: instRpms, test_tag: tags, ftest_arg: getFunctionalArgs( pragma_suffix: pragma_suffix, nvme: nvme, default_nvme: default_nvme, provider: provider)['ftest_arg'], - test_function: 'runTestFunctionalV2')) + test_function: 'runTestFunctionalV2', + coverage_stash: coverage_stash)) } finally { println("[${name}] Running functionalTestPostV2()") functionalTestPostV2() diff --git a/vars/runTestFunctionalV2.groovy b/vars/runTestFunctionalV2.groovy index 985f21b46..5315408b0 100644 --- a/vars/runTestFunctionalV2.groovy +++ b/vars/runTestFunctionalV2.groovy @@ -88,14 +88,21 @@ Map call(Map config = [:]) { // Restore the ignore failure setting config['ignore_failure'] = ignore_failure - String coverageFile = 'test.cov' - if (!fileExists('test.cov')) { - coverageFile += '_not_done' - fileOperations([fileCreateOperation(fileName: coverageFile, - fileContent: '')]) + // Stash code coverage file if it exists + String coverage_stash = config.get('coverage_stash', '') + String coverage_dir = "${basedir}bullseye_coverage_logs" + String coverage_file = "${coverage_dir}/test.cov" + if (coverage_stash) { + if (fileExists(coverage_file)) { + // String name = 'func' + stage_info['pragma_suffix'] + '-cov' + println("[${env.STAGE_NAME}] Stashing ${coverage_file} in ${coverage_stash}") + dir(coverage_dir) { + stash name: coverage_stash, includes: 'test.cov' + } + } else { + println("[${env.STAGE_NAME}] No ${coverage_file} file found for ${coverage_stash}!") + } } - String name = 'func' + stage_info['pragma_suffix'] + '-cov' - stash name: config.get('coverage_stash', name), - includes: coverageFile + return runData } diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index 8d10191c4..8c6f91de0 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -94,7 +94,7 @@ Map afterTest(Map config, Map testRunInfo) { } else { result['result'] = checkJunitFiles(testResults: testResults) } - if (config['with_valgrind'] || config['NLT']) { + if (config['check_valgrind_errors']) { vgrcs = sh label: 'Check for Valgrind errors', script: "grep -E ')' ${valgrind_pattern} || true", returnStdout: true @@ -129,31 +129,40 @@ Map call(Map config = [:]) { long startDate = System.currentTimeMillis() String nodelist = config.get('NODELIST', env.NODELIST) String test_script = config.get('test_script', 'ci/unit/test_main.sh') - Map stage_info = parseStageInfo(config) String inst_rpms = config.get('inst_rpms', '') - if (stage_info['compiler'] == 'covc') { - if (stage_info['java_pkg']) { - inst_rpms += " ${stage_info['java_pkg']}" - } - } + // Support backwards compatibility with parseStageInfo when config keys are ommitted + Map stage_info = parseStageInfo(config) + Integer node_count = config.get('node_count', stage_info['node_count']) + String target = config.get('target', stage_info['ci_target']) + String distro_version = config.get('distro_version', stage_info['distro_version']) + String compiler = config.get('compiler', stage_info['compiler']) + String build_type = config.get('build_type', stage_info['build_type']) + String with_valgrind = config.get('with_valgrind', stage_info.get('with_valgrind', '')) + Boolean NLT = config.get('NLT', stage_info.get('NLT', false)) + String always_script = config.get( + 'always_script', stage_info.get('always_script', 'ci/unit/test_post_always.sh')) + String valgrind_pattern = config.get( + 'valgrind_pattern', stage_info.get('valgrind_pattern', 'unit-test-*memcheck.xml')) + String test_results = config.get( + 'test_results', stage_info.get('testResults', 'test_results/*.xml')) String image_version = config.get('image_version', '') ?: - (stage_info['ci_target'] =~ /([a-z]+)(.*)/)[0][1] + stage_info['distro_version'] + (target =~ /([a-z]+)(.*)/)[0][1] + distro_version Map runData = provisionNodes( NODELIST: nodelist, - node_count: stage_info['node_count'], + node_count: node_count, distro: image_version, inst_repos: config.get('inst_repos', ''), inst_rpms: inst_rpms) /* el9-gcc-tests */ - String target_stash = (image_version ?: ${stage_info['target']}).split('\\.')[0] + String target_stash = (image_version ?: target).split('\\.')[0] - target_stash += '-' + stage_info['compiler'] - if (stage_info['build_type']) { - target_stash += '-' + stage_info['build_type'] + target_stash += '-' + compiler + if (build_type) { + target_stash += '-' + build_type } List stashes = [] @@ -171,51 +180,41 @@ Map call(Map config = [:]) { } } - if (stage_info['compiler'] == 'covc') { - String tools_url = env.JENKINS_URL + - 'job/daos-stack/job/tools/job/master' + - '/lastSuccessfulBuild/artifact/' - httpRequest url: tools_url + 'bullseyecoverage-linux.tar', - httpMode: 'GET', - outputFile: 'bullseye.tar' - } - - String with_valgrind = stage_info.get('with_valgrind', '') - Map p = [:] - p['stashes'] = stashes - p['script'] = "SSH_KEY_ARGS=${env.SSH_KEY_ARGS} " + - "NODELIST=${nodelist} " + - "WITH_VALGRIND=${with_valgrind} " + - test_script - p['junit_files'] = config.get('junit_files', 'test_results/*.xml') - p['context'] = config.get('context', 'test/' + env.STAGE_NAME) - p['description'] = config.get('description', env.STAGE_NAME) + Map params = [:] + params['stashes'] = stashes + params['script'] = "SSH_KEY_ARGS=${env.SSH_KEY_ARGS} " + + "NODELIST=${nodelist} " + + "WITH_VALGRIND=${with_valgrind} " + + test_script + params['junit_files'] = config.get('junit_files', 'test_results/*.xml') + params['context'] = config.get('context', 'test/' + env.STAGE_NAME) + params['description'] = config.get('description', env.STAGE_NAME) // Do not let runTest abort the pipeline as want artifact/log collection. - p['ignore_failure'] = true + params['ignore_failure'] = true // runTest no longer knows now to notify for Unit Tests - p['notify_result'] = false + params['notify_result'] = false int time = config.get('timeout_time', 120) as int String unit = config.get('timeout_unit', 'MINUTES') Map runTestData = [:] timeout(time: time, unit: unit) { - runTestData = runTest p + runTestData = runTest params runTestData.each { resultKey, data -> runData[resultKey] = data } } - p['always_script'] = stage_info.get('always_script', - 'ci/unit/test_post_always.sh') - p['valgrind_pattern'] = stage_info.get('valgrind_pattern', - 'unit-test-*memcheck.xml') - p['testResults'] = stage_info.get('testResults', 'test_results/*.xml') - p['with_valgrind'] = with_valgrind - p['NLT'] = stage_info['NLT'] - runTestData = afterTest(p, runData) + params['always_script'] = always_script + params['valgrind_pattern'] = valgrind_pattern + params['testResults'] = test_results + params['check_valgrind_errors'] = (with_valgrind || NLT) && (compiler != 'covc') + runTestData = afterTest(params, runData) runTestData.each { resultKey, data -> runData[resultKey] = data } - if (stage_info['compiler'] == 'covc') { + if (compiler == 'covc') { + // Stash the bullseye code coverage report if it was generated stash name: config.get('coverage_stash', "${target_stash}-unit-cov"), - includes: 'test.cov' + includes: '**/test.cov' + allowEmpty: true } + int runTime = durationSeconds(startDate) runData['unittest_time'] = runTime @@ -229,11 +228,5 @@ Map call(Map config = [:]) { stash name: results_map, includes: results_map - // Stash any optional test coverage reports for the stage - String code_coverage = 'code_coverage_' + sanitizedStageName() - stash name: code_coverage, - includes: '**/code_coverage.json', - allowEmpty: true - return runData } diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index aa081e008..331233d68 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -30,9 +30,19 @@ void call(Map config = [:]) { Map stage_info = parseStageInfo(config) String cbcResult = currentBuild.currentResult + // Support backwards compatibility with parseStageInfo when config keys are ommitted + String target = config.get('target', stage_info['ci_target']) + String compiler = config.get('compiler', stage_info['compiler']) + String build_type = config.get('build_type', stage_info['build_type']) + String with_valgrind = config.get('with_valgrind', stage_info.get('with_valgrind', '')) + String valgrind_pattern = config.get( + 'valgrind_pattern', stage_info.get('valgrind_pattern', 'unit-test-*memcheck.xml')) + String testResults = config.get( + 'testResults', stage_info.get('testResults', 'test_results/*.xml')) + Boolean NLT = config.get('NLT', stage_info.get('NLT', false)) + Boolean check_valgrind_errors = (with_valgrind || NLT) && (compiler != 'covc') + // Stash the Valgrind files for later analysis - String valgrind_pattern = stage_info.get('valgrind_pattern', - 'unit-test-*memcheck.xml') if (config['valgrind_stash']) { try { stash name: config['valgrind_stash'], includes: valgrind_pattern @@ -56,7 +66,6 @@ void call(Map config = [:]) { List artifact_list = config.get('artifacts', ['run_test.sh/*']) - String testResults = stage_info.get('testResults', 'test_results/*.xml') if (testResults != 'None' ) { // groovylint-disable-next-line NoDouble double health_scale = 1.0 @@ -66,7 +75,7 @@ void call(Map config = [:]) { junit testResults: testResults, healthScaleFactor: health_scale } - if (stage_info['with_valgrind'] || stage_info['NLT']) { + if (check_valgrind_errors) { String suite = sanitizedStageName() int vgfail = 0 String testdata @@ -97,16 +106,16 @@ void call(Map config = [:]) { archiveArtifacts artifacts: artifactPat, allowEmptyArchive: results['ignore_failure'] } - String target_stash = "${stage_info['target']}-${stage_info['compiler']}" - if (stage_info['build_type']) { - target_stash += '-' + stage_info['build_type'] + String target_stash = "${target}-${compiler}" + if (build_type) { + target_stash += "-${build_type}" } // Coverage instrumented tests and Valgrind are probably mutually exclusive - if (stage_info['compiler'] == 'covc') { + if (compiler == 'covc') { return } - if (stage_info['NLT']) { + if (NLT) { String cb_result = currentBuild.result discoverGitReferenceBuild(referenceJob: config.get('referenceJobName', 'daos-stack/daos/master'),