feat: #811 Add tracebility comments in Effective Set

build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/pojo/bom/ApplicationBomDTO.java

@@ -38,7 +38,7 @@ public class ApplicationBomDTO {
     private Map<String, Map<String, Object>> smartplugs;
     private Map<String, Map<String, Object>> cdn;
     private Map<String, Map<String, Object>> sampleRepo;
-    private Map<String, String> deployParams;
+    private Map<String, Object> deployParams;
 
     //deployment-descriptor params
     private Map<String, Map<String, Object>> deployDescriptors;

build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/pojo/bom/EntitiesMap.java

@@ -39,7 +39,7 @@ public class EntitiesMap {
     public Map<String, Map<String, Object>> deployDescParamsMap = new TreeMap<>();
     public Map<String, Map<String, Object>> commonParamsMap = new TreeMap<>();
     public Map<String, Map<String, Object>> perServiceParams = new TreeMap<>();
-    public Map<String, String> deployParams = new TreeMap<>();
+    public Map<String, Object> deployParams = new TreeMap<>();
     public String deployerSessionId;
     public String appChartName;
 }

build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/ParameterUtils.java

@@ -105,6 +105,55 @@ private static Parameter copyOldValues(Parameter original, Object newValue) {
                 .build();
     }
 
+    /**
+     * Recursively copies maps into {@link TreeMap} key order (same stable ordering as
+     * {@code convertParameterMapToObject}), but keeps {@link Parameter} wrappers and their metadata.
+     * Nested map/list values inside a {@link Parameter} are sorted and reassigned via {@link #copyOldValues(Parameter, Object)}.
+     */
+    public static Map<String, Object> deepSortMapKeysPreservingParameters(Map<?, ?> source) {
+        if (source == null || source.isEmpty()) {
+            return new TreeMap<>();
+        }
+        List<Map.Entry<?, ?>> entries = new ArrayList<>(source.entrySet());
+        entries.sort(Comparator.comparing(e -> String.valueOf(e.getKey())));
+        Map<String, Object> out = new TreeMap<>();
+        for (Map.Entry<?, ?> e : entries) {
+            out.put(String.valueOf(e.getKey()), deepSortValuePreservingParameters(e.getValue()));
+        }
+        return out;
+    }
+
+    private static Object deepSortValuePreservingParameters(Object value) {
+        if (value instanceof Parameter p) {
+            Object inner = p.getValue();
+            if (inner instanceof Map<?, ?>) {
+                return copyOldValues(p, deepSortMapKeysPreservingParameters((Map<?, ?>) inner));
+            }
+            if (inner instanceof List<?>) {
+                return copyOldValues(p, deepSortListPreservingParameters((List<?>) inner));
+            }
+            return p;
+        }
+        if (value instanceof Map<?, ?>) {
+            return deepSortMapKeysPreservingParameters((Map<?, ?>) value);
+        }
+        if (value instanceof List<?>) {
+            return deepSortListPreservingParameters((List<?>) value);
+        }
+        return value;
+    }
+
+    private static List<Object> deepSortListPreservingParameters(List<?> list) {
+        if (list == null) {
+            return null;
+        }
+        List<Object> out = new ArrayList<>(list.size());
+        for (Object item : list) {
+            out.add(deepSortValuePreservingParameters(item));
+        }
+        return out;
+    }
+
     public static void splitBgDomainParams(Map<String, Object> bgDomainMap,
                                            Map<String, Object> bgDomainSecureMap,
                                            Map<String, Object> bgDomainParamsMap) {
@@ -137,4 +186,66 @@ private static void updateParameter(Map<String, Parameter> customParams, Map<Str
             params.remove(key);
         }
     }
+
+    @SuppressWarnings("unchecked")
+    public static void wrapPlainMapWithOrigin(Map<String, Object> map, String origin) {
+        if (map == null || map.isEmpty()) {
+            return;
+        }
+        for (Map.Entry<String, Object> entry : map.entrySet()) {
+            Object value = entry.getValue();
+            if (value instanceof Parameter) {
+                ((Parameter) value).setOrigin(origin);
+            } else if (value instanceof Map) {
+                wrapPlainMapWithOrigin((Map<String, Object>) value, origin);
+            } else if (value instanceof List) {
+                wrapPlainListWithOrigin((List<Object>) value, origin);
+            } else {
+                entry.setValue(new Parameter(value, origin, false));
+            }
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public static List<Object> wrapPlainListWithOrigin(List<Object> list, String origin) {
+        if (list == null || list.isEmpty()) {
+            return list;
+        }
+        for (int i = 0; i < list.size(); i++) {
+            Object item = list.get(i);
+            if (item instanceof Parameter) {
+                // Keep existing Parameter as is
+            } else if (item instanceof Map) {
+                wrapPlainMapWithOrigin((Map<String, Object>) item, origin);
+            } else if (item instanceof List) {
+                list.set(i, wrapPlainListWithOrigin((List<Object>) item, origin));
+            } else {
+                list.set(i, new Parameter(item, origin, false));
+            }
+        }
+        return list;
+    }
+
+
+    public static Map<String, Object> unwrapParameterValues(
+            Map<String, Object> inputMap) {
+        if (inputMap == null) {
+            return Collections.emptyMap();
+        }
+
+        Map<String, Object> result = new HashMap<>();
+
+        for (Map.Entry<String, Object> entry : inputMap.entrySet()) {
+            Object value = entry.getValue();
+
+            if (value instanceof Parameter) {
+                Parameter param = (Parameter) value;
+                result.put(entry.getKey(), param.getValue());
+            } else {
+                result.put(entry.getKey(), value);
+            }
+        }
+        return result;
+    }
+
 }

build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/constant/CredentialConstants.java

@@ -17,11 +17,12 @@
 package org.qubership.cloud.devops.commons.utils.constant;
 
 public class CredentialConstants {
+    public static final String DEFAULT_EMPTY_STRING = "";
     public static final String CALCULABLE_CREDS_FIELD = "#creds";
     public static final String CALCULABLE_CLOUD_CREDS_FIELD = "#credscl";
     public static final String CALCULABLE_NS_CREDS_FIELD = "#credsns";
-    public static final String DEFAULT_DBAAS_AGGREGATOR_LOGIN = "";
-    public static final String DEFAULT_DBAAS_AGGREGATOR_PASSWORD = "";
-    public static final String DEFAULT_MAAS_LOGIN = "";
-    public static final String DEFAULT_MAAS_PASSWORD = "";
+    public static final String DEFAULT_DBAAS_AGGREGATOR_LOGIN = DEFAULT_EMPTY_STRING;
+    public static final String DEFAULT_DBAAS_AGGREGATOR_PASSWORD = DEFAULT_EMPTY_STRING;
+    public static final String DEFAULT_MAAS_LOGIN = DEFAULT_EMPTY_STRING;
+    public static final String DEFAULT_MAAS_PASSWORD = DEFAULT_EMPTY_STRING;
 }

build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/constant/ParametersConstants.java

@@ -18,28 +18,37 @@
 
 
 public class ParametersConstants {
-    public static final String TENANT_ORIGIN = "Env/Tenant: %s",
+    public static final String TENANT_ORIGIN = "tenant",
             TENANT_E2E_ORIGIN = "Env/Tenant/e2e: %s",
             TENANT_PARAMETER_SET_ORIGIN = "Params/Tenant: %s/%s",
             TENANT_CONFIG_SERVER_ORIGIN = "Env/Tenant/config-server: %s",
             PARAMETER_SET_ORIGIN = "Params: %s",
             PARAMETER_SET_APP_ORIGIN = "Params/App: %s/%s",
-            CLOUD_ORIGIN = "Env/Cloud: %s/%s",
+            CLOUD_ORIGIN = "cloud",
             CLOUD_E2E_ORIGIN = "Env/Cloud: %s/%s",
             CLOUD_CONFIG_SERVER_ORIGIN = "Env/Cloud/config-server: %s/%s",
             CLOUD_PARAMETER_SET_ORIGIN = "Params/Cloud: %s/%s/%s",
             CLOUD_PARAMETER_SET_APP_ORIGIN = "Params/Cloud/Apps: %s/%s/%s/%s",
             CLOUD_APP_ORIGIN = "Env/Cloud/App: %s/%s/%s",
             CLOUD_APP_CONFIG_SERVER_ORIGIN = "Env/Cloud/App/config-server: %s/%s/%s",
-            NS_ORIGIN = "Env/Namespace: %s/%s/%s",
+            NS_ORIGIN = "namespace",
             NS_E2E_ORIGIN = "Env/Namespace/e2e: %s/%s/%s",
             NS_CONFIG_SERVER_ORIGIN = "Env/Namespace/config-server: %s/%s/%s",
             NS_APP_ORIGIN = "Env/Namespace/App: %s/%s/%s/%s",
             NS_APP_CONFIG_SERVER_ORIGIN = "Env/Namespace/App/config-server: %s/%s/%s/%s",
             NS_PARAMETER_SET_ORIGIN = "Params/Namespace: %s/%s/%s/%s",
             NS_PARAMETER_SET_APP_ORIGIN = "Params/Namespace/Apps: %s/%s/%s/%s/%s",
-            APP_ORIGIN = "Application: %s",
-            CUSTOM_PARAMS_ORIGIN = "Custom",
+            APP_ORIGIN = "application",
+            CUSTOM_PARAMS_ORIGIN = "custom params",
             STRUCTURED_GLOBAL_RESOURCE_PROFILE = "STRUCTURED_GLOBAL_RESOURCE_PROFILE",
+            BG_DOMAIN = "bg-domain",
+            ENVGENE_DEFAULT = "envgene default",
+            ENVGENE_PIPELINE_PARAMETER =  "envgene pipeline parameter",
+            ENVGENE_PIPELINE_CONSUMER_PARAMETER =  "consumer params",
+            ENVGENE_CALCULATED = "envgene calculated",
+            SBOM_ORIGIN ="sbom",
+            COMPOSITE_STRUCTURE = "composite-structure",
+            RP_OVERRIDE_ORIGIN = "rp-override: %s",
+            RP_BASELINE_ORIGIN = "rp-baseline: %s",
             GLOBAL_RESOURCE_PROFILE = "GLOBAL_RESOURCE_PROFILE";
 }

build_effective_set_generator/effective-set-generator/pom.xml

@@ -45,6 +45,12 @@
         </dependencies>
     </dependencyManagement>
     <dependencies>
+        <dependency>
+            <groupId>org.snakeyaml</groupId>
+            <artifactId>snakeyaml-engine</artifactId>
+            <version>3.0.1</version>
+            <scope>compile</scope>
+        </dependency>
         <dependency>
             <groupId>org.yaml</groupId>
             <artifactId>snakeyaml</artifactId>

build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/CmdbCli.java

@@ -121,6 +121,7 @@ private void setSharedData() {
         sharedData.setPcsspPaths(envParams.pcssp != null ? List.of(envParams.pcssp) : new ArrayList<>());
         sharedData.setAppChartValidation(envParams.appChartValidation);
         prepareCustomParameters(getCustomParams(envParams.customParams));
+        sharedData.setEnableTraceability(envParams.enableTraceability);
         populateDeploymentSessionId(envParams.extraParams);
     }
 
@@ -201,6 +202,9 @@ static class EnvCommandSpace {
         @CommandLine.Option(names = {"-cp", "--custom-params"}, description = "Custom Parameters")
         String customParams;
 
+        @CommandLine.Option(names = {"-etr", "--enable-traceability"}, description = "Enable traceability by including parameter origin information in output files (true/false)", arity = "1", defaultValue = "false")
+        boolean enableTraceability = false;
+
     }
 
 }

build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/parser/CliParameterParser.java

@@ -63,6 +63,8 @@
 import static org.qubership.cloud.devops.cli.exceptions.constants.ExceptionMessage.APP_PROCESS_FAILED;
 import static org.qubership.cloud.devops.commons.exceptions.constant.ExceptionAdditionalInfoMessages.ENTITY_NOT_FOUND;
 import static org.qubership.cloud.devops.commons.utils.ConsoleLogger.*;
+import static org.qubership.cloud.devops.commons.utils.ParameterUtils.*;
+import static org.qubership.cloud.devops.commons.utils.constant.ParametersConstants.*;
 
 @Dependent
 @Slf4j
@@ -104,15 +106,17 @@ private void processAndSaveParameters(Optional<SolutionBomDTO> solutionDescripto
         Map<String, Object> runtimeMappingFileData = new ConcurrentHashMap<>();
         Map<String, Object> cleanupMappingFileData = new ConcurrentHashMap<>();
         Map<String, String> errorList = new ConcurrentHashMap<>();
-        Map<String, String> k8TokenMap = new ConcurrentHashMap<>();
+        Map<String, Object> k8TokenMap = new ConcurrentHashMap<>();
         namespaceDTOMap.keySet().parallelStream().forEach(namespaceName -> {
             String originalNamespace = inputData.getNamespaceDTOMap().get(namespaceName).getName();
             String credentialsId = findDefaultCredentialsId(namespaceName);
+            String credentialsIdOrigin = !StringUtils.isEmpty(inputData.getNamespaceDTOMap().get(namespaceName).getCredentialsId()) ?
+                        NS_ORIGIN : CLOUD_ORIGIN;
             if (StringUtils.isNotEmpty(credentialsId)) {
                 CredentialDTO credentialDTO = inputData.getCredentialDTOMap().get(credentialsId);
                 if (credentialDTO != null) {
                     SecretCredentialsDTO secCred = (SecretCredentialsDTO) credentialDTO.getData();
-                    k8TokenMap.put(originalNamespace, secCred.getSecret());
+                    k8TokenMap.put(originalNamespace, new Parameter(secCred.getSecret(),credentialsIdOrigin,false));
                 }
             }
         });
@@ -170,7 +174,7 @@ private void processAndSaveParameters(Optional<SolutionBomDTO> solutionDescripto
 
     }
 
-    private void generateE2EOutput(String tenantName, String cloudName, Map<String, String> k8TokenMap) throws IOException {
+    private void generateE2EOutput(String tenantName, String cloudName, Map<String, Object> k8TokenMap) throws IOException {
         ParameterBundle parameterBundle = parametersServiceV2.getCliE2EParameter(tenantName, cloudName);
         if (parameterBundle.getE2eParams() == null) {
             parameterBundle.setE2eParams(new HashMap<>());
@@ -197,20 +201,24 @@ private void processBgDomainParameters() {
         }
     }
 
-    private void createTopologyFiles(Map<String, String> k8TokenMap) throws IOException {
+    private void createTopologyFiles(Map<String, Object> k8TokenMap) throws IOException {
         Map<String, Object> topologyParams = new TreeMap<>();
         Map<String, Object> topologySecuredParams = new TreeMap<>();
         Map<String, Object> clusterParameterMap = getClusterMap();
-        topologyParams.put("composite_structure", getObjectMap(inputData.getCompositeStructureDTO()));
-        topologyParams.put("environments", inputData.getClusterMap());
-        topologyParams.put("cluster", clusterParameterMap);
-        topologySecuredParams.put("k8s_tokens", k8TokenMap);
+        Map<String, Object> compositeStructure = getObjectMap(inputData.getCompositeStructureDTO());
+        topologyParams.put("composite_structure", new Parameter(compositeStructure,ParametersConstants.COMPOSITE_STRUCTURE,false));
+        Map<String, Object>  environments = inputData.getClusterMap();
+        topologyParams.put("environments",  new Parameter(environments,ENVGENE_CALCULATED,false));
+        topologyParams.put("cluster", new Parameter(clusterParameterMap,ENVGENE_CALCULATED,false));
+
+        topologySecuredParams.put("k8s_tokens", new Parameter(unwrapParameterValues(k8TokenMap),ENVGENE_CALCULATED,false));
+
         Map<String, Object> bgDomainMap = getObjectMap(inputData.getBgDomainEntityDTO());
         Map<String, Object> bgDomainSecureMap = new LinkedHashMap<>();
         Map<String, Object> bgDomainParamsMap = new LinkedHashMap<>();
         ParameterUtils.splitBgDomainParams(bgDomainMap, bgDomainSecureMap, bgDomainParamsMap);
-        topologySecuredParams.put("bg_domain", bgDomainSecureMap);
-        topologyParams.put("bg_domain", bgDomainParamsMap);
+        topologySecuredParams.put("bg_domain", new Parameter(bgDomainSecureMap,BG_DOMAIN,false));
+        topologyParams.put("bg_domain", new Parameter(bgDomainParamsMap,BG_DOMAIN,false));
         String topologyDir = String.format("%s/%s", sharedData.getOutputDir(), "topology");
         fileDataConverter.writeToFile(topologyParams, topologyDir, "parameters.yaml");
         fileDataConverter.writeToFile(topologySecuredParams, topologyDir, "credentials.yaml");
@@ -249,7 +257,7 @@ private void createPipelineFiles(ParameterBundle parameterBundle) {
                     }
                 }
                 if (obj == null && StringUtils.isNotEmpty(k.getValue())) {
-                    consumerParamsMap.put(k.getName(), k.getValue());
+                    consumerParamsMap.put(k.getName(), new Parameter(k.getValue(), ENVGENE_PIPELINE_CONSUMER_PARAMETER,false));
                 }
                 if (obj == null && StringUtils.isEmpty(k.getValue()) && k.isRequired()) {
                     throw new ConsumerFileProcessingException("Property " + k + " is required and no value is defined in E2E configurations");
@@ -271,7 +279,7 @@ private void createE2EFiles(ParameterBundle parameterBundle) throws IOException
     }
 
     public void generateOutput(String tenantName, String cloudName, String namespaceName, String appName,
-                               String appVersion, String appFileRef, Map<String, String> k8TokenMap) throws IOException {
+                               String appVersion, String appFileRef, Map<String, Object> k8TokenMap) throws IOException {
         DeployerInputs deployerInputs = DeployerInputs.builder().appVersion(appVersion).appFileRef(appFileRef).build();
         String originalNamespace = inputData.getNamespaceDTOMap().get(namespaceName).getName();
         ParameterBundle parameterBundle;
@@ -353,7 +361,8 @@ private void createFiles(String namespaceName, String appName, ParameterBundle p
             //deployment
             fileDataConverter.writeToFile(parameterBundle.getDeployParams(), deploymentDir, "deployment-parameters.yaml");
             if (StringUtils.isNotBlank(parameterBundle.getAppChartName())) {
-                fileDataConverter.writeToFile(parameterBundle.getPerServiceParams(), appChartPath.toString(), "deployment-parameters.yaml");
+                Map<String, Object> perServiceParams = parameterBundle.getPerServiceParams();
+                fileDataConverter.writeToFile(perServiceParams, appChartPath.toString(), "deployment-parameters.yaml");
             }
             fileDataConverter.writeToFile(parameterBundle.getCollisionSecureParameters(), deploymentDir, "collision-credentials.yaml");
             fileDataConverter.writeToFile(parameterBundle.getCollisionDeployParameters(), deploymentDir, "collision-deployment-parameters.yaml");

build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/pojo/dto/shared/SharedData.java

@@ -57,4 +57,6 @@ public class SharedData {
     @Builder.Default
     private Map<String, Object> customRuntimeParamMap = Collections.emptyMap();
 
+    private boolean enableTraceability;
+
 }