Feature/tir rm dyn data add helm status

CHANGELOG.md

@@ -15,6 +15,7 @@
 * Enhance SSDS Document Generation Performance using New Atlassian APIs ([#1084](https://github.com/opendevstack/ods-jenkins-shared-library/issues/1084))
 * Simplify successor management since now issue links are no longer inherited ([#1116](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1116))
 * Sort unit test keys associated with requirements in TCR ([#1192](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1116))
+* TIR - remove dynamic pod data, surface helm status and helm report tables reformatting ([#1143](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1143))
 
 ### Fixed
 * Fix Tailor deployment drifts for D, Q envs ([#1055](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1055))

src/org/ods/component/AbstractDeploymentStrategy.groovy

@@ -14,6 +14,9 @@ abstract class AbstractDeploymentStrategy implements IDeploymentStrategy {
     @Override
     abstract Map<String, List<PodData>> deploy()
 
+    // Fetches original kubernetes revisions of deployment resources.
+    //
+    // returns a two level map with keys resourceKind -> resourceName -> revision (int)
     protected Map<String, Map<String, Integer>> fetchOriginalVersions(Map<String, List<String>> deploymentResources) {
         def originalVersions = [:]
         deploymentResources.each { resourceKind, resourceNames ->

src/org/ods/component/Context.groovy

@@ -1,17 +1,18 @@
 package org.ods.component
 
-import org.ods.util.ShellWithRetry
-import org.ods.util.Logger
+import com.cloudbees.groovy.cps.NonCPS
+import groovy.json.JsonOutput
+import groovy.json.JsonSlurperClassic
 import org.ods.services.BitbucketService
 import org.ods.services.GitService
 import org.ods.services.NexusService
 import org.ods.services.OpenShiftService
-import com.cloudbees.groovy.cps.NonCPS
-import groovy.json.JsonSlurperClassic
-import groovy.json.JsonOutput
-import java.util.concurrent.ExecutionException
+import org.ods.util.ILogger
 import org.ods.util.IPipelineSteps
 import org.ods.util.PipelineSteps
+import org.ods.util.ShellWithRetry
+
+import java.util.concurrent.ExecutionException
 
 @SuppressWarnings(['MethodCount', 'UnnecessaryObjectReferences'])
 class Context implements IContext {
@@ -25,7 +26,7 @@ class Context implements IContext {
     // config is a map of config properties to customise the behaviour.
     private final Map config
     private final IPipelineSteps steps
-    private final Logger logger
+    private final ILogger logger
     // artifact store, the interface to MRP
     private final def artifactUriStore = [builds: [:], deployments: [:]]
 
@@ -34,7 +35,7 @@ class Context implements IContext {
 
     private String appDomain
 
-    Context(def script, Map config, Logger logger, boolean localCheckoutEnabled = true) {
+    Context(def script, Map config, ILogger logger, boolean localCheckoutEnabled = true) {
         this.script = script
         this.config = config
         this.logger = logger

src/org/ods/component/HelmDeploymentStrategy.groovy

@@ -1,38 +1,35 @@
 package org.ods.component
 
 import com.cloudbees.groovy.cps.NonCPS
-import groovy.json.JsonOutput
 import groovy.transform.TypeChecked
 import groovy.transform.TypeCheckingMode
 import org.ods.services.JenkinsService
 import org.ods.services.OpenShiftService
+import org.ods.util.HelmStatus
 import org.ods.util.ILogger
-import org.ods.util.PipelineSteps
+import org.ods.util.IPipelineSteps
 import org.ods.util.PodData
 
 class HelmDeploymentStrategy extends AbstractDeploymentStrategy {
 
     // Constructor arguments
-    private final Script script
     private final IContext context
     private final OpenShiftService openShift
     private final JenkinsService jenkins
     private final ILogger logger
-
+    private final IPipelineSteps steps
     // assigned in constructor
-    private def steps
     private final RolloutOpenShiftDeploymentOptions options
 
     @SuppressWarnings(['AbcMetric', 'CyclomaticComplexity', 'ParameterCount'])
     HelmDeploymentStrategy(
-        def script,
+        IPipelineSteps steps,
         IContext context,
         Map<String, Object> config,
         OpenShiftService openShift,
         JenkinsService jenkins,
         ILogger logger
     ) {
-
         if (!config.selector) {
             config.selector = context.selector
         }
@@ -73,10 +70,9 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy {
         if (!config.helmPrivateKeyCredentialsId) {
             config.helmPrivateKeyCredentialsId = "${context.cdProject}-helm-private-key"
         }
-        this.script = script
         this.context = context
         this.logger = logger
-        this.steps = new PipelineSteps(script)
+        this.steps = steps
 
         this.options = new RolloutOpenShiftDeploymentOptions(config)
         this.openShift = openShift
@@ -93,23 +89,19 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy {
         // Tag images which have been built in this pipeline from cd project into target project
         retagImages(context.targetProject, getBuiltImages())
 
-        logger.info "Rolling out ${context.componentId} with HELM, selector: ${options.selector}"
+        logger.info("Rolling out ${context.componentId} with HELM, selector: ${options.selector}")
         helmUpgrade(context.targetProject)
-
-        def deploymentResources = openShift.getResourcesForComponent(
-            context.targetProject, DEPLOYMENT_KINDS, options.selector
-        )
-        logger.info("${this.class.name} -- DEPLOYMENT RESOURCES")
-        logger.info(
-            JsonOutput.prettyPrint(
-                JsonOutput.toJson(deploymentResources)))
+        HelmStatus helmStatus = openShift.helmStatus(context.targetProject, options.helmReleaseName)
+        if (logger.debugMode) {
+            def helmStatusMap = helmStatus.toMap()
+            logger.debug("${this.class.name} -- HELM STATUS: ${helmStatusMap}")
+        }
 
         // // FIXME: pauseRollouts is non trivial to determine!
         // // we assume that Helm does "Deployment" that should work for most
         // // cases since they don't have triggers.
         // metadataSvc.updateMetadata(false, deploymentResources)
-        def rolloutData = getRolloutData(deploymentResources) ?: [:]
-        logger.info(JsonOutput.prettyPrint(JsonOutput.toJson(rolloutData)))
+        def rolloutData = getRolloutData(helmStatus)
         return rolloutData
     }
 
@@ -125,7 +117,7 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy {
                 options.helmValues['componentId'] = context.componentId
 
                 // we persist the original ones set from outside - here we just add ours
-                Map mergedHelmValues = [:]
+                Map<String, String> mergedHelmValues = [:]
                 mergedHelmValues << options.helmValues
 
                 // we add the global ones - this allows usage in subcharts
@@ -142,13 +134,14 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy {
 
                 // deal with dynamic value files - which are env dependent
                 def mergedHelmValuesFiles = []
-                mergedHelmValuesFiles.addAll(options.helmValuesFiles)
 
-                options.helmEnvBasedValuesFiles.each { envValueFile ->
-                    mergedHelmValuesFiles << envValueFile.replace('.env',
-                        ".${context.environment}")
+                def  envConfigFiles = options.helmEnvBasedValuesFiles.collect { filenamePattern ->
+                    filenamePattern.replace('.env.', ".${context.environment}.")
                 }
 
+                mergedHelmValuesFiles.addAll(options.helmValuesFiles)
+                mergedHelmValuesFiles.addAll(envConfigFiles)
+
                 openShift.helmUpgrade(
                     targetProject,
                     options.helmReleaseName,
@@ -169,33 +162,51 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy {
     // ]
     @TypeChecked(TypeCheckingMode.SKIP)
     private Map<String, List<PodData>> getRolloutData(
-        Map<String, List<String>> deploymentResources) {
+        HelmStatus helmStatus
+    ) {
+        Map<String, List<PodData>> rolloutData = [:]
 
-        def rolloutData = [:]
-        deploymentResources.each { resourceKind, resourceNames ->
-            resourceNames.each { resourceName ->
-                def podData = []
+        Map<String, List<String>> deploymentKinds = helmStatus.resourcesByKind
+                .findAll { kind, res -> kind in DEPLOYMENT_KINDS }
+
+        deploymentKinds.each { kind, names ->
+            names.each { name ->
+                context.addDeploymentToArtifactURIs("${name}-deploymentMean",
+                    [
+                        type: 'helm',
+                        selector: options.selector,
+                        namespace: context.targetProject,
+                        chartDir: options.chartDir,
+                        helmReleaseName: options.helmReleaseName,
+                        helmEnvBasedValuesFiles: options.helmEnvBasedValuesFiles,
+                        helmValuesFiles: options.helmValuesFiles,
+                        helmValues: options.helmValues,
+                        helmDefaultFlags: options.helmDefaultFlags,
+                        helmAdditionalFlags: options.helmAdditionalFlags,
+                        helmStatus: helmStatus.toMap(),
+                    ]
+                )
+                def podDataContext = [
+                    "targetProject=${context.targetProject}",
+                    "selector=${options.selector}",
+                    "name=${name}",
+                ]
+                def msgPodsNotFound = "Could not find 'running' pod(s) for '${podDataContext.join(', ')}'"
+                List<PodData> podData = null
                 for (def i = 0; i < options.deployTimeoutRetries; i++) {
-                    podData = openShift.checkForPodData(context.targetProject, options.selector, resourceName)
-                    if (!podData.isEmpty()) {
+                    podData = openShift.checkForPodData(context.targetProject, options.selector, name)
+                    if (podData) {
                         break
                     }
-                    steps.echo("Could not find 'running' pod(s) with label '${options.selector}' - waiting")
+                    steps.echo("${msgPodsNotFound} - waiting")
                     steps.sleep(12)
                 }
-                context.addDeploymentToArtifactURIs("${resourceName}-deploymentMean",
-                    [
-                        'type': 'helm',
-                        'selector': options.selector,
-                        'chartDir': options.chartDir,
-                        'helmReleaseName': options.helmReleaseName,
-                        'helmEnvBasedValuesFiles': options.helmEnvBasedValuesFiles,
-                        'helmValuesFiles': options.helmValuesFiles,
-                        'helmValues': options.helmValues,
-                        'helmDefaultFlags': options.helmDefaultFlags,
-                        'helmAdditionalFlags': options.helmAdditionalFlags,
-                    ])
-                rolloutData["${resourceKind}/${resourceName}"] = podData
+                if (!podData) {
+                    throw new RuntimeException(msgPodsNotFound)
+                }
+                logger.debug("Helm podData for ${podDataContext.join(', ')}: ${podData}")
+
+                rolloutData["${kind}/${name}"] = podData
 
                 // We need to find the pod that was created as a result of the deployment.
                 // The previous pod may still be alive when we use a rollout strategy.
@@ -211,14 +222,11 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy {
                     throw new RuntimeException(
                         "Unable to determine the most recent Pod. " +
                         "Multiple pods running with the same latest creation timestamp " +
-                        "and different images found for ${resourceName}"
+                        "and different images found for ${name}"
                     )
                 }
-                // TODO: Once the orchestration pipeline can deal with multiple replicas,
-                // update this to store multiple pod artifacts.
-                // TODO: Potential conflict if resourceName is duplicated between
                 // Deployment and DeploymentConfig resource.
-                context.addDeploymentToArtifactURIs(resourceName, latestPods[0]?.toMap())
+                context.addDeploymentToArtifactURIs(name, latestPods[0]?.toMap())
             }
         }
         return rolloutData
@@ -299,4 +307,5 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy {
         }
         return equal
     }
+
 }

src/org/ods/component/RolloutOpenShiftDeploymentStage.groovy

@@ -2,9 +2,10 @@ package org.ods.component
 
 import groovy.transform.TypeChecked
 import groovy.transform.TypeCheckingMode
-import org.ods.services.OpenShiftService
 import org.ods.services.JenkinsService
+import org.ods.services.OpenShiftService
 import org.ods.util.ILogger
+import org.ods.util.PipelineSteps
 
 @SuppressWarnings('ParameterCount')
 @TypeChecked
@@ -110,26 +111,27 @@ class RolloutOpenShiftDeploymentStage extends Stage {
         // We have to move everything here,
         // otherwise Jenkins will complain
         // about: "hudson.remoting.ProxyException: CpsCallableInvocation{methodName=fileExists, ..."
-        def isHelmDeployment = steps.fileExists(options.chartDir + '/Chart.yaml')
+        def isHelmDeployment = this.steps.fileExists(options.chartDir + '/Chart.yaml')
         logger.info("isHelmDeployment: ${isHelmDeployment}")
-        def isTailorDeployment = steps.fileExists(options.openshiftDir)
+        def isTailorDeployment = this.steps.fileExists(options.openshiftDir)
 
         if (isTailorDeployment && isHelmDeployment) {
-            steps.error("Must be either a Tailor based deployment or a Helm based deployment")
+            this.steps.error("Must be either a Tailor based deployment or a Helm based deployment")
             throw new IllegalStateException("Must be either a Tailor based deployment or a Helm based deployment")
         }
 
         // Use tailorDeployment in the following cases:
         // (1) We have an openshiftDir
         // (2) We do not have an openshiftDir but neither do we have an indication that it is Helm
+        def steps = new PipelineSteps(script)
         if (isTailorDeployment || (!isHelmDeployment && !isTailorDeployment)) {
-            deploymentStrategy = new TailorDeploymentStrategy(script, context, config, openShift, jenkins, logger)
+            deploymentStrategy = new TailorDeploymentStrategy(steps, context, config, openShift, jenkins, logger)
             String resourcePath = 'org/ods/component/RolloutOpenShiftDeploymentStage.deprecate-tailor.GString.txt'
-            def msg =steps.libraryResource(resourcePath)
+            def msg = this.steps.libraryResource(resourcePath)
             logger.warn(msg)
         }
         if (isHelmDeployment) {
-            deploymentStrategy = new HelmDeploymentStrategy(script, context, config, openShift, jenkins, logger)
+            deploymentStrategy = new HelmDeploymentStrategy(steps, context, config, openShift, jenkins, logger)
         }
         logger.info("deploymentStrategy: ${deploymentStrategy} -- ${deploymentStrategy.class.name}")
         return deploymentStrategy.deploy()

src/org/ods/component/TailorDeploymentStrategy.groovy

@@ -5,25 +5,24 @@ import groovy.transform.TypeCheckingMode
 import org.ods.services.JenkinsService
 import org.ods.services.OpenShiftService
 import org.ods.util.ILogger
-import org.ods.util.PipelineSteps
+import org.ods.util.IPipelineSteps
 import org.ods.util.PodData
 
 class TailorDeploymentStrategy extends AbstractDeploymentStrategy {
 
     // Constructor arguments
-    private final Script script
     private final IContext context
     private final OpenShiftService openShift
     private final JenkinsService jenkins
     private final ILogger logger
+    private final IPipelineSteps steps
 
     // assigned in constructor
-    private def steps
     private final RolloutOpenShiftDeploymentOptions options
 
     @SuppressWarnings(['AbcMetric', 'CyclomaticComplexity', 'ParameterCount'])
     TailorDeploymentStrategy(
-        def script,
+        IPipelineSteps steps,
         IContext context,
         Map<String, Object> config,
         OpenShiftService openShift,
@@ -67,10 +66,9 @@ class TailorDeploymentStrategy extends AbstractDeploymentStrategy {
         if (!config.containsKey('tailorParams')) {
             config.tailorParams = []
         }
-        this.script = script
         this.context = context
         this.logger = logger
-        this.steps = new PipelineSteps(script)
+        this.steps = steps
 
         this.options = new RolloutOpenShiftDeploymentOptions(config)
         this.openShift = openShift