diff --git a/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/pojo/bom/ApplicationBomDTO.java b/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/pojo/bom/ApplicationBomDTO.java index af6b6e92a..3fe5bc6f8 100644 --- a/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/pojo/bom/ApplicationBomDTO.java +++ b/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/pojo/bom/ApplicationBomDTO.java @@ -38,7 +38,7 @@ public class ApplicationBomDTO { private Map> smartplugs; private Map> cdn; private Map> sampleRepo; - private Map deployParams; + private Map deployParams; //deployment-descriptor params private Map> deployDescriptors; diff --git a/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/pojo/bom/EntitiesMap.java b/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/pojo/bom/EntitiesMap.java index 30bb991d6..43123d381 100644 --- a/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/pojo/bom/EntitiesMap.java +++ b/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/pojo/bom/EntitiesMap.java @@ -39,7 +39,7 @@ public class EntitiesMap { public Map> deployDescParamsMap = new TreeMap<>(); public Map> commonParamsMap = new TreeMap<>(); public Map> perServiceParams = new TreeMap<>(); - public Map deployParams = new TreeMap<>(); + public Map deployParams = new TreeMap<>(); public String deployerSessionId; public String appChartName; } diff --git a/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/ParameterUtils.java b/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/ParameterUtils.java index 399ab3ace..7a0840822 100644 --- a/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/ParameterUtils.java +++ b/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/ParameterUtils.java @@ -105,6 +105,55 @@ private static Parameter copyOldValues(Parameter original, Object newValue) { .build(); } + /** + * Recursively copies maps into {@link TreeMap} key order (same stable ordering as + * {@code convertParameterMapToObject}), but keeps {@link Parameter} wrappers and their metadata. + * Nested map/list values inside a {@link Parameter} are sorted and reassigned via {@link #copyOldValues(Parameter, Object)}. + */ + public static Map deepSortMapKeysPreservingParameters(Map source) { + if (source == null || source.isEmpty()) { + return new TreeMap<>(); + } + List> entries = new ArrayList<>(source.entrySet()); + entries.sort(Comparator.comparing(e -> String.valueOf(e.getKey()))); + Map out = new TreeMap<>(); + for (Map.Entry e : entries) { + out.put(String.valueOf(e.getKey()), deepSortValuePreservingParameters(e.getValue())); + } + return out; + } + + private static Object deepSortValuePreservingParameters(Object value) { + if (value instanceof Parameter p) { + Object inner = p.getValue(); + if (inner instanceof Map) { + return copyOldValues(p, deepSortMapKeysPreservingParameters((Map) inner)); + } + if (inner instanceof List) { + return copyOldValues(p, deepSortListPreservingParameters((List) inner)); + } + return p; + } + if (value instanceof Map) { + return deepSortMapKeysPreservingParameters((Map) value); + } + if (value instanceof List) { + return deepSortListPreservingParameters((List) value); + } + return value; + } + + private static List deepSortListPreservingParameters(List list) { + if (list == null) { + return null; + } + List out = new ArrayList<>(list.size()); + for (Object item : list) { + out.add(deepSortValuePreservingParameters(item)); + } + return out; + } + public static void splitBgDomainParams(Map bgDomainMap, Map bgDomainSecureMap, Map bgDomainParamsMap) { @@ -137,4 +186,66 @@ private static void updateParameter(Map customParams, Map map, String origin) { + if (map == null || map.isEmpty()) { + return; + } + for (Map.Entry entry : map.entrySet()) { + Object value = entry.getValue(); + if (value instanceof Parameter) { + ((Parameter) value).setOrigin(origin); + } else if (value instanceof Map) { + wrapPlainMapWithOrigin((Map) value, origin); + } else if (value instanceof List) { + wrapPlainListWithOrigin((List) value, origin); + } else { + entry.setValue(new Parameter(value, origin, false)); + } + } + } + + @SuppressWarnings("unchecked") + public static List wrapPlainListWithOrigin(List list, String origin) { + if (list == null || list.isEmpty()) { + return list; + } + for (int i = 0; i < list.size(); i++) { + Object item = list.get(i); + if (item instanceof Parameter) { + // Keep existing Parameter as is + } else if (item instanceof Map) { + wrapPlainMapWithOrigin((Map) item, origin); + } else if (item instanceof List) { + list.set(i, wrapPlainListWithOrigin((List) item, origin)); + } else { + list.set(i, new Parameter(item, origin, false)); + } + } + return list; + } + + + public static Map unwrapParameterValues( + Map inputMap) { + if (inputMap == null) { + return Collections.emptyMap(); + } + + Map result = new HashMap<>(); + + for (Map.Entry entry : inputMap.entrySet()) { + Object value = entry.getValue(); + + if (value instanceof Parameter) { + Parameter param = (Parameter) value; + result.put(entry.getKey(), param.getValue()); + } else { + result.put(entry.getKey(), value); + } + } + return result; + } + } diff --git a/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/constant/CredentialConstants.java b/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/constant/CredentialConstants.java index 181beebbe..2a1c04835 100644 --- a/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/constant/CredentialConstants.java +++ b/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/constant/CredentialConstants.java @@ -17,11 +17,12 @@ package org.qubership.cloud.devops.commons.utils.constant; public class CredentialConstants { + public static final String DEFAULT_EMPTY_STRING = ""; public static final String CALCULABLE_CREDS_FIELD = "#creds"; public static final String CALCULABLE_CLOUD_CREDS_FIELD = "#credscl"; public static final String CALCULABLE_NS_CREDS_FIELD = "#credsns"; - public static final String DEFAULT_DBAAS_AGGREGATOR_LOGIN = ""; - public static final String DEFAULT_DBAAS_AGGREGATOR_PASSWORD = ""; - public static final String DEFAULT_MAAS_LOGIN = ""; - public static final String DEFAULT_MAAS_PASSWORD = ""; + public static final String DEFAULT_DBAAS_AGGREGATOR_LOGIN = DEFAULT_EMPTY_STRING; + public static final String DEFAULT_DBAAS_AGGREGATOR_PASSWORD = DEFAULT_EMPTY_STRING; + public static final String DEFAULT_MAAS_LOGIN = DEFAULT_EMPTY_STRING; + public static final String DEFAULT_MAAS_PASSWORD = DEFAULT_EMPTY_STRING; } diff --git a/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/constant/ParametersConstants.java b/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/constant/ParametersConstants.java index fa8b863be..af77bfe23 100644 --- a/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/constant/ParametersConstants.java +++ b/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/constant/ParametersConstants.java @@ -18,28 +18,37 @@ public class ParametersConstants { - public static final String TENANT_ORIGIN = "Env/Tenant: %s", + public static final String TENANT_ORIGIN = "tenant", TENANT_E2E_ORIGIN = "Env/Tenant/e2e: %s", TENANT_PARAMETER_SET_ORIGIN = "Params/Tenant: %s/%s", TENANT_CONFIG_SERVER_ORIGIN = "Env/Tenant/config-server: %s", PARAMETER_SET_ORIGIN = "Params: %s", PARAMETER_SET_APP_ORIGIN = "Params/App: %s/%s", - CLOUD_ORIGIN = "Env/Cloud: %s/%s", + CLOUD_ORIGIN = "cloud", CLOUD_E2E_ORIGIN = "Env/Cloud: %s/%s", CLOUD_CONFIG_SERVER_ORIGIN = "Env/Cloud/config-server: %s/%s", CLOUD_PARAMETER_SET_ORIGIN = "Params/Cloud: %s/%s/%s", CLOUD_PARAMETER_SET_APP_ORIGIN = "Params/Cloud/Apps: %s/%s/%s/%s", CLOUD_APP_ORIGIN = "Env/Cloud/App: %s/%s/%s", CLOUD_APP_CONFIG_SERVER_ORIGIN = "Env/Cloud/App/config-server: %s/%s/%s", - NS_ORIGIN = "Env/Namespace: %s/%s/%s", + NS_ORIGIN = "namespace", NS_E2E_ORIGIN = "Env/Namespace/e2e: %s/%s/%s", NS_CONFIG_SERVER_ORIGIN = "Env/Namespace/config-server: %s/%s/%s", NS_APP_ORIGIN = "Env/Namespace/App: %s/%s/%s/%s", NS_APP_CONFIG_SERVER_ORIGIN = "Env/Namespace/App/config-server: %s/%s/%s/%s", NS_PARAMETER_SET_ORIGIN = "Params/Namespace: %s/%s/%s/%s", NS_PARAMETER_SET_APP_ORIGIN = "Params/Namespace/Apps: %s/%s/%s/%s/%s", - APP_ORIGIN = "Application: %s", - CUSTOM_PARAMS_ORIGIN = "Custom", + APP_ORIGIN = "application", + CUSTOM_PARAMS_ORIGIN = "custom params", STRUCTURED_GLOBAL_RESOURCE_PROFILE = "STRUCTURED_GLOBAL_RESOURCE_PROFILE", + BG_DOMAIN = "bg-domain", + ENVGENE_DEFAULT = "envgene default", + ENVGENE_PIPELINE_PARAMETER = "envgene pipeline parameter", + ENVGENE_PIPELINE_CONSUMER_PARAMETER = "consumer params", + ENVGENE_CALCULATED = "envgene calculated", + SBOM_ORIGIN ="sbom", + COMPOSITE_STRUCTURE = "composite-structure", + RP_OVERRIDE_ORIGIN = "rp-override: %s", + RP_BASELINE_ORIGIN = "rp-baseline: %s", GLOBAL_RESOURCE_PROFILE = "GLOBAL_RESOURCE_PROFILE"; } diff --git a/build_effective_set_generator/effective-set-generator/pom.xml b/build_effective_set_generator/effective-set-generator/pom.xml index 83773a1bb..e5fbafe24 100644 --- a/build_effective_set_generator/effective-set-generator/pom.xml +++ b/build_effective_set_generator/effective-set-generator/pom.xml @@ -45,6 +45,12 @@ + + org.snakeyaml + snakeyaml-engine + 3.0.1 + compile + org.yaml snakeyaml diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/CmdbCli.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/CmdbCli.java index 42b7dd5e8..fec80f588 100644 --- a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/CmdbCli.java +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/CmdbCli.java @@ -121,6 +121,7 @@ private void setSharedData() { sharedData.setPcsspPaths(envParams.pcssp != null ? List.of(envParams.pcssp) : new ArrayList<>()); sharedData.setAppChartValidation(envParams.appChartValidation); prepareCustomParameters(getCustomParams(envParams.customParams)); + sharedData.setEnableTraceability(envParams.enableTraceability); populateDeploymentSessionId(envParams.extraParams); } @@ -201,6 +202,9 @@ static class EnvCommandSpace { @CommandLine.Option(names = {"-cp", "--custom-params"}, description = "Custom Parameters") String customParams; + @CommandLine.Option(names = {"-etr", "--enable-traceability"}, description = "Enable traceability by including parameter origin information in output files (true/false)", arity = "1", defaultValue = "false") + boolean enableTraceability = false; + } } diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/parser/CliParameterParser.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/parser/CliParameterParser.java index 19c15529f..7661ba2a4 100644 --- a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/parser/CliParameterParser.java +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/parser/CliParameterParser.java @@ -63,6 +63,8 @@ import static org.qubership.cloud.devops.cli.exceptions.constants.ExceptionMessage.APP_PROCESS_FAILED; import static org.qubership.cloud.devops.commons.exceptions.constant.ExceptionAdditionalInfoMessages.ENTITY_NOT_FOUND; import static org.qubership.cloud.devops.commons.utils.ConsoleLogger.*; +import static org.qubership.cloud.devops.commons.utils.ParameterUtils.*; +import static org.qubership.cloud.devops.commons.utils.constant.ParametersConstants.*; @Dependent @Slf4j @@ -104,15 +106,17 @@ private void processAndSaveParameters(Optional solutionDescripto Map runtimeMappingFileData = new ConcurrentHashMap<>(); Map cleanupMappingFileData = new ConcurrentHashMap<>(); Map errorList = new ConcurrentHashMap<>(); - Map k8TokenMap = new ConcurrentHashMap<>(); + Map k8TokenMap = new ConcurrentHashMap<>(); namespaceDTOMap.keySet().parallelStream().forEach(namespaceName -> { String originalNamespace = inputData.getNamespaceDTOMap().get(namespaceName).getName(); String credentialsId = findDefaultCredentialsId(namespaceName); + String credentialsIdOrigin = !StringUtils.isEmpty(inputData.getNamespaceDTOMap().get(namespaceName).getCredentialsId()) ? + NS_ORIGIN : CLOUD_ORIGIN; if (StringUtils.isNotEmpty(credentialsId)) { CredentialDTO credentialDTO = inputData.getCredentialDTOMap().get(credentialsId); if (credentialDTO != null) { SecretCredentialsDTO secCred = (SecretCredentialsDTO) credentialDTO.getData(); - k8TokenMap.put(originalNamespace, secCred.getSecret()); + k8TokenMap.put(originalNamespace, new Parameter(secCred.getSecret(),credentialsIdOrigin,false)); } } }); @@ -170,7 +174,7 @@ private void processAndSaveParameters(Optional solutionDescripto } - private void generateE2EOutput(String tenantName, String cloudName, Map k8TokenMap) throws IOException { + private void generateE2EOutput(String tenantName, String cloudName, Map k8TokenMap) throws IOException { ParameterBundle parameterBundle = parametersServiceV2.getCliE2EParameter(tenantName, cloudName); if (parameterBundle.getE2eParams() == null) { parameterBundle.setE2eParams(new HashMap<>()); @@ -197,20 +201,24 @@ private void processBgDomainParameters() { } } - private void createTopologyFiles(Map k8TokenMap) throws IOException { + private void createTopologyFiles(Map k8TokenMap) throws IOException { Map topologyParams = new TreeMap<>(); Map topologySecuredParams = new TreeMap<>(); Map clusterParameterMap = getClusterMap(); - topologyParams.put("composite_structure", getObjectMap(inputData.getCompositeStructureDTO())); - topologyParams.put("environments", inputData.getClusterMap()); - topologyParams.put("cluster", clusterParameterMap); - topologySecuredParams.put("k8s_tokens", k8TokenMap); + Map compositeStructure = getObjectMap(inputData.getCompositeStructureDTO()); + topologyParams.put("composite_structure", new Parameter(compositeStructure,ParametersConstants.COMPOSITE_STRUCTURE,false)); + Map environments = inputData.getClusterMap(); + topologyParams.put("environments", new Parameter(environments,ENVGENE_CALCULATED,false)); + topologyParams.put("cluster", new Parameter(clusterParameterMap,ENVGENE_CALCULATED,false)); + + topologySecuredParams.put("k8s_tokens", new Parameter(unwrapParameterValues(k8TokenMap),ENVGENE_CALCULATED,false)); + Map bgDomainMap = getObjectMap(inputData.getBgDomainEntityDTO()); Map bgDomainSecureMap = new LinkedHashMap<>(); Map bgDomainParamsMap = new LinkedHashMap<>(); ParameterUtils.splitBgDomainParams(bgDomainMap, bgDomainSecureMap, bgDomainParamsMap); - topologySecuredParams.put("bg_domain", bgDomainSecureMap); - topologyParams.put("bg_domain", bgDomainParamsMap); + topologySecuredParams.put("bg_domain", new Parameter(bgDomainSecureMap,BG_DOMAIN,false)); + topologyParams.put("bg_domain", new Parameter(bgDomainParamsMap,BG_DOMAIN,false)); String topologyDir = String.format("%s/%s", sharedData.getOutputDir(), "topology"); fileDataConverter.writeToFile(topologyParams, topologyDir, "parameters.yaml"); fileDataConverter.writeToFile(topologySecuredParams, topologyDir, "credentials.yaml"); @@ -249,7 +257,7 @@ private void createPipelineFiles(ParameterBundle parameterBundle) { } } if (obj == null && StringUtils.isNotEmpty(k.getValue())) { - consumerParamsMap.put(k.getName(), k.getValue()); + consumerParamsMap.put(k.getName(), new Parameter(k.getValue(), ENVGENE_PIPELINE_CONSUMER_PARAMETER,false)); } if (obj == null && StringUtils.isEmpty(k.getValue()) && k.isRequired()) { throw new ConsumerFileProcessingException("Property " + k + " is required and no value is defined in E2E configurations"); @@ -271,7 +279,7 @@ private void createE2EFiles(ParameterBundle parameterBundle) throws IOException } public void generateOutput(String tenantName, String cloudName, String namespaceName, String appName, - String appVersion, String appFileRef, Map k8TokenMap) throws IOException { + String appVersion, String appFileRef, Map k8TokenMap) throws IOException { DeployerInputs deployerInputs = DeployerInputs.builder().appVersion(appVersion).appFileRef(appFileRef).build(); String originalNamespace = inputData.getNamespaceDTOMap().get(namespaceName).getName(); ParameterBundle parameterBundle; @@ -353,7 +361,8 @@ private void createFiles(String namespaceName, String appName, ParameterBundle p //deployment fileDataConverter.writeToFile(parameterBundle.getDeployParams(), deploymentDir, "deployment-parameters.yaml"); if (StringUtils.isNotBlank(parameterBundle.getAppChartName())) { - fileDataConverter.writeToFile(parameterBundle.getPerServiceParams(), appChartPath.toString(), "deployment-parameters.yaml"); + Map perServiceParams = parameterBundle.getPerServiceParams(); + fileDataConverter.writeToFile(perServiceParams, appChartPath.toString(), "deployment-parameters.yaml"); } fileDataConverter.writeToFile(parameterBundle.getCollisionSecureParameters(), deploymentDir, "collision-credentials.yaml"); fileDataConverter.writeToFile(parameterBundle.getCollisionDeployParameters(), deploymentDir, "collision-deployment-parameters.yaml"); diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/pojo/dto/shared/SharedData.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/pojo/dto/shared/SharedData.java index 46f22f750..59c96b97b 100644 --- a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/pojo/dto/shared/SharedData.java +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/pojo/dto/shared/SharedData.java @@ -57,4 +57,6 @@ public class SharedData { @Builder.Default private Map customRuntimeParamMap = Collections.emptyMap(); + private boolean enableTraceability; + } diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/repository/implementation/FileDataConverterImpl.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/repository/implementation/FileDataConverterImpl.java index 06007809e..43f23baf6 100644 --- a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/repository/implementation/FileDataConverterImpl.java +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/repository/implementation/FileDataConverterImpl.java @@ -21,8 +21,8 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.cyclonedx.model.Bom; import org.qubership.cloud.devops.cli.exceptions.constants.ExceptionMessage; -import org.qubership.cloud.devops.cli.utils.FileSystemUtils; import org.qubership.cloud.devops.cli.utils.deserializer.BomMixin; +import org.qubership.cloud.devops.cli.utils.yaml.YamlFileWriter; import org.qubership.cloud.devops.commons.exceptions.FileParseException; import org.qubership.cloud.devops.commons.exceptions.JsonParseException; import org.qubership.cloud.devops.commons.repository.interfaces.FileDataConverter; @@ -34,12 +34,10 @@ import org.yaml.snakeyaml.nodes.Node; import org.yaml.snakeyaml.nodes.Tag; import org.yaml.snakeyaml.representer.Representer; -import org.qubership.cloud.devops.cli.utils.yaml.AdaptiveYaml; import java.io.*; import java.util.Base64; import java.util.Map; -import java.util.TreeMap; import static org.qubership.cloud.devops.commons.utils.ConsoleLogger.*; @@ -49,11 +47,11 @@ public class FileDataConverterImpl implements FileDataConverter { public static final String CLEANUPER = "cleanuper"; private final ObjectMapper objectMapper; - private final FileSystemUtils fileSystemUtils; + private final YamlFileWriter yamlFileWriter; @Inject - public FileDataConverterImpl(FileSystemUtils fileSystemUtils) { - this.fileSystemUtils = fileSystemUtils; + public FileDataConverterImpl(YamlFileWriter yamlFileWriter) { + this.yamlFileWriter = yamlFileWriter; this.objectMapper = new ObjectMapper(new YAMLFactory()); } @@ -96,19 +94,7 @@ public T parseInputFile(TypeReference typeReference, File file) { @Override public void writeToFile(Map params, String... args) throws IOException { - File file = fileSystemUtils.getFileFromGivenPath(args); - - boolean expand = params != null && !params.isEmpty() - && AdaptiveYaml.shouldExpand(params); - if (expand) { - logInfo("removing anchors and aliases for file: " + file.getAbsolutePath()); - } - - try (BufferedWriter writer = new BufferedWriter(new FileWriter(file))) { - if (params != null && !params.isEmpty()) { - getYamlObject(expand).dump(params, writer); - } - } + yamlFileWriter.write(params, args); } @@ -125,7 +111,7 @@ private static Yaml getYamlObject(boolean expand) { options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); options.setDefaultScalarStyle(DumperOptions.ScalarStyle.PLAIN); options.setPrettyFlow(false); - if (expand) { + if (expand) { options.setDereferenceAliases(true); } Representer representer = new Representer(options) { diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/service/implementation/ProfileServiceCliImpl.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/service/implementation/ProfileServiceCliImpl.java index 1ef018837..5f4a95797 100644 --- a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/service/implementation/ProfileServiceCliImpl.java +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/service/implementation/ProfileServiceCliImpl.java @@ -26,6 +26,8 @@ import org.qubership.cloud.devops.commons.service.interfaces.TenantConfigurationService; import org.qubership.cloud.devops.commons.pojo.profile.dto.ProfileFullDto; import org.qubership.cloud.devops.commons.pojo.profile.model.Profile; +import org.qubership.cloud.devops.commons.utils.Parameter; +import org.qubership.cloud.devops.commons.utils.constant.ParametersConstants; import org.qubership.cloud.devops.commons.utils.mapper.ProfileMapper; import org.qubership.cloud.devops.commons.service.interfaces.ProfileService; import jakarta.enterprise.context.ApplicationScoped; @@ -34,7 +36,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; @Slf4j @ApplicationScoped @@ -76,6 +77,7 @@ public Profile getProfileByTenant(String tenantName, String profileName) { public void setOverrideProfiles(String appName, String serviceName, Profile overrideProfile, Map profileValues) { expandDottedKeys(profileValues); if (overrideProfile != null) { + String overrideOrigin = String.format(ParametersConstants.RP_OVERRIDE_ORIGIN, overrideProfile.getName()); ApplicationProfile override = overrideProfile.getApplications().stream() .filter(app -> appName.equals(app.getName())) .findFirst() @@ -87,7 +89,7 @@ public void setOverrideProfiles(String appName, String serviceName, Profile over .findFirst().orElse(null); if (serviceOverride != null) { for (ParameterProfile param : serviceOverride.getParameters()) { - putNestedValue(profileValues, param.getName(), param.getValue()); + putNestedValue(profileValues, param.getName(), new Parameter(param.getValue(), overrideOrigin,false)); } } } diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/BomReaderUtilsImplV2.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/BomReaderUtilsImplV2.java index b8977971c..b9dc8b431 100644 --- a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/BomReaderUtilsImplV2.java +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/BomReaderUtilsImplV2.java @@ -29,6 +29,7 @@ import org.cyclonedx.model.component.data.Content; import org.qubership.cloud.devops.cli.exceptions.MandatoryParameterException; import org.qubership.cloud.devops.cli.pojo.dto.shared.SharedData; +import org.qubership.cloud.devops.commons.utils.Parameter; import org.qubership.cloud.devops.commons.exceptions.AppChartValidationException; import org.qubership.cloud.devops.commons.exceptions.BomProcessingException; import org.qubership.cloud.devops.commons.pojo.bom.ApplicationBomDTO; @@ -39,12 +40,15 @@ import org.qubership.cloud.devops.commons.service.interfaces.ProfileService; import org.qubership.cloud.devops.commons.service.interfaces.RegistryConfigurationService; import org.qubership.cloud.devops.commons.utils.ServiceArtifactType; +import org.qubership.cloud.devops.commons.utils.constant.ParametersConstants; import java.io.File; import java.util.*; import java.util.regex.Pattern; +import static org.qubership.cloud.devops.commons.utils.ParameterUtils.wrapPlainMapWithOrigin; import static org.qubership.cloud.devops.commons.utils.constant.ApplicationConstants.*; +import static org.qubership.cloud.devops.commons.utils.constant.ParametersConstants.*; @ApplicationScoped @Slf4j @@ -72,6 +76,8 @@ public ApplicationBomDTO getAppServicesWithProfiles(String appName, String appFi if (bomContent == null) { return null; } + String baselineOrigin = String.format(ParametersConstants.RP_BASELINE_ORIGIN, baseline); + EntitiesMap entitiesMap = new EntitiesMap(); try { Component component = bomContent.getMetadata().getComponent(); @@ -112,14 +118,15 @@ public ApplicationBomDTO getAppServicesWithProfiles(String appName, String appFi private void populateEntityDeployDescParams(EntitiesMap entitiesMap, List components, Bom bomContent) { Map commonParamsMap = new TreeMap<>(); - commonParamsMap.put("APPLICATION_NAME", bomContent.getMetadata().getComponent().getName()); - commonParamsMap.put("MANAGED_BY", "argocd"); + String appName = bomContent.getMetadata().getComponent().getName(); + commonParamsMap.put("APPLICATION_NAME", new Parameter(appName, SBOM_ORIGIN, false)); + commonParamsMap.put("MANAGED_BY", new Parameter("argocd", ENVGENE_DEFAULT, false)); if (StringUtils.isNotEmpty(sharedData.getDeploymentSessionId())) { - commonParamsMap.put("DEPLOYMENT_SESSION_ID", sharedData.getDeploymentSessionId()); + commonParamsMap.put("DEPLOYMENT_SESSION_ID", new Parameter(sharedData.getDeploymentSessionId(), ENVGENE_PIPELINE_PARAMETER, false)); entitiesMap.setDeployerSessionId(sharedData.getDeploymentSessionId()); } else { String deployerSessionId = UUID.randomUUID().toString(); - commonParamsMap.put("DEPLOYMENT_SESSION_ID", deployerSessionId); + commonParamsMap.put("DEPLOYMENT_SESSION_ID", new Parameter(deployerSessionId, ENVGENE_CALCULATED, false)); entitiesMap.setDeployerSessionId(deployerSessionId); } for (Component component : components) { @@ -138,34 +145,37 @@ private void processConfigServiceComponentDeployDescParams(Map primaryArtifactMap = new TreeMap<>(); - List> artifacts = new ArrayList<>(); + List artifacts = new ArrayList<>(); Map tArtifactMap = new TreeMap<>(); if (CollectionUtils.isNotEmpty(component.getComponents())) { for (Component subComponent : component.getComponents()) { entity = "sub component '" + subComponent.getName() + "' of service:" + component.getName(); if (subComponent.getMimeType().equalsIgnoreCase(serviceArtifactType.getArtifactMimeType())) { - populateOptionalParam(primaryArtifactMap, "artifactId", subComponent.getName()); - populateOptionalParam(primaryArtifactMap, "groupId", subComponent.getGroup()); - populateOptionalParam(primaryArtifactMap, "version", subComponent.getVersion()); + primaryArtifactMap.put("artifactId", new Parameter(subComponent.getName(), SBOM_ORIGIN, false)); + primaryArtifactMap.put("groupId", new Parameter(subComponent.getGroup(), SBOM_ORIGIN, false)); + primaryArtifactMap.put("version", new Parameter(subComponent.getVersion(), SBOM_ORIGIN, false)); } if (SUB_SERVICE_ARTIFACT_MIME_TYPES.contains(subComponent.getMimeType())) { String name = checkIfMandatory(subComponent.getName(), "name", entity); String version = checkIfMandatory(subComponent.getVersion(), "version", entity); Map artifactMap = new TreeMap<>(); - artifactMap.put("artifact_id", ""); - artifactMap.put("artifact_path", ""); - artifactMap.put("artifact_type", ""); - artifactMap.put("classifier", getPropertyValue(subComponent, "classifier", null, true, entity)); - artifactMap.put("deploy_params", ""); - artifactMap.put("gav", ""); - artifactMap.put("group_id", ""); - artifactMap.put("id", checkIfMandatory(subComponent.getGroup(), "group", entity) + ":" + name + ":" + version); - artifactMap.put("name", name + "-" + version + "." + - getPropertyValue(subComponent, "type", null, true, entity)); - artifactMap.put("repository", ""); - artifactMap.put("type", getPropertyValue(subComponent, "type", null, true, entity)); - artifactMap.put("url", ""); - artifactMap.put("version", ""); + artifactMap.put("artifact_id", new Parameter("", ENVGENE_DEFAULT, false)); + artifactMap.put("artifact_path", new Parameter("", ENVGENE_DEFAULT, false)); + artifactMap.put("artifact_type", new Parameter("", ENVGENE_DEFAULT, false)); + Parameter classifierParam = getPropertyValueAsParameter(subComponent, "classifier", null, true, entity, SBOM_ORIGIN); + if (classifierParam != null) { + artifactMap.put("classifier", classifierParam); + } + artifactMap.put("deploy_params", new Parameter("", ENVGENE_DEFAULT, false)); + artifactMap.put("gav", new Parameter("", ENVGENE_DEFAULT, false)); + artifactMap.put("group_id", new Parameter("", ENVGENE_DEFAULT, false)); + artifactMap.put("id", new Parameter(checkIfMandatory(subComponent.getGroup(), "group", entity) + ":" + name + ":" + version, ENVGENE_CALCULATED, false)); + String typeValue = getPropertyValue(subComponent, "type", null, true, entity); + artifactMap.put("name", new Parameter(name + "-" + version + "." + typeValue, ENVGENE_CALCULATED, false)); + artifactMap.put("repository", new Parameter("", ENVGENE_DEFAULT, false)); + artifactMap.put("type", new Parameter(typeValue, SBOM_ORIGIN, false)); + artifactMap.put("url", new Parameter("", ENVGENE_DEFAULT, false)); + artifactMap.put("version", new Parameter("", ENVGENE_DEFAULT, false)); artifacts.add(artifactMap); } if (APPLICATION_ZIP.equalsIgnoreCase(subComponent.getMimeType())) { @@ -173,9 +183,9 @@ private void processConfigServiceComponentDeployDescParams(Map()); - deployDescParams.put("docker_registry", getPropertyValue(component, "docker_registry", null, true, entity)); - deployDescParams.put("full_image_name", getPropertyValue(component, "full_image_name", null, true, entity)); - deployDescParams.put("git_branch", getPropertyValue(component, "git_branch", null, true, entity)); - deployDescParams.put("git_revision", getPropertyValue(component, "git_revision", null, true, entity)); - deployDescParams.put("git_url", getPropertyValue(component, "git_url", null, true, entity)); - deployDescParams.put("image", getPropertyValue(component, "full_image_name", null, true, entity)); - deployDescParams.put("image_type", getPropertyValue(component, "image_type", null, true, entity)); - deployDescParams.put("name", checkIfMandatory(component.getName(), "name", entity)); - deployDescParams.put("promote_artifacts", getPropertyValue(component, "promote_artifacts", null, true, entity)); - deployDescParams.put("qualifier", getPropertyValue(component, "qualifier", null, true, entity)); - deployDescParams.put("version", checkIfMandatory(component.getVersion(), "version", entity)); + deployDescParams.put("deploy_param", getPropertyValueAsParameter(component, "deploy_param", "", true, entity, SBOM_ORIGIN)); + deployDescParams.put("artifacts", new Parameter(new ArrayList<>(),ENVGENE_DEFAULT,false)); + deployDescParams.put("docker_registry", getPropertyValueAsParameter(component, "docker_registry", null, true, entity, SBOM_ORIGIN)); + Parameter fullImageNameParam = getPropertyValueAsParameter(component, "full_image_name", null, true, entity, SBOM_ORIGIN); + deployDescParams.put("full_image_name", fullImageNameParam); + deployDescParams.put("image", fullImageNameParam); + deployDescParams.put("git_branch", getPropertyValueAsParameter(component, "git_branch", null, true, entity, SBOM_ORIGIN)); + deployDescParams.put("git_revision", getPropertyValueAsParameter(component, "git_revision", null, true, entity, SBOM_ORIGIN)); + deployDescParams.put("git_url", getPropertyValueAsParameter(component, "git_url", null, true, entity, SBOM_ORIGIN)); + deployDescParams.put("image_type", getPropertyValueAsParameter(component, "image_type", null, true, entity, SBOM_ORIGIN)); + deployDescParams.put("name", checkIfMandatoryAsParameter(component.getName(), "name", entity, SBOM_ORIGIN)); + deployDescParams.put("promote_artifacts", getPropertyValueAsParameter(component, "promote_artifacts", null, true, entity, SBOM_ORIGIN)); + deployDescParams.put("qualifier", getPropertyValueAsParameter(component, "qualifier", null, true, entity, SBOM_ORIGIN)); + deployDescParams.put("version", checkIfMandatoryAsParameter(component.getVersion(), "version", entity, SBOM_ORIGIN)); deployParamsMap.put(component.getName(), deployDescParams); } @@ -247,6 +262,22 @@ private String getPropertyValue(Component component, String propertyName, String return result; } + private Parameter getPropertyValueAsParameter(Component component, String propertyName, String defaultValue, boolean mandatory, String entity, String origin) { + String value = component.getProperties().stream() + .filter(p -> propertyName.equals(p.getName())) + .map(Property::getValue) + .findFirst() + .orElse(null); + if (value != null) { + return new Parameter(value, origin, false); + } + if (mandatory) { + String fallback = checkIfMandatory(defaultValue, propertyName, entity); + return new Parameter(fallback, ENVGENE_DEFAULT, false); + } + return null; + } + private String checkIfMandatory(String value, String propertyName, String entity) { if (value == null) { throw new MandatoryParameterException(String.format("Mandatory Parameter '%s' is not present in '%s'.", propertyName, entity)); @@ -254,6 +285,11 @@ private String checkIfMandatory(String value, String propertyName, String entity return value; } + private Parameter checkIfMandatoryAsParameter(String value, String propertyName, String entity, String origin) { + checkIfMandatory(value, propertyName, entity); + return new Parameter(value, origin, false); + } + private void getPerServiceEntities(EntitiesMap entitiesMap, List components, String appName, String baseline, Profile override, Bom bomContent) { for (Component component : components) { if (IMAGE_SERVICE_MIME_TYPES.contains(component.getMimeType())) { @@ -278,12 +314,11 @@ private void processConfigServiceComponent(Map> serv Map profileValues = new TreeMap<>(); Map serviceParams = new TreeMap<>(); String entity = "service:" + component.getName(); - serviceParams.put("ARTIFACT_DESCRIPTOR_VERSION", checkIfMandatory(bomContent.getMetadata().getComponent().getVersion(), "version in metadata", entity)); - serviceParams.put("DEPLOYMENT_RESOURCE_NAME", checkIfMandatory(component.getName(), "name", entity) + "-v1"); - serviceParams.put("DEPLOYMENT_VERSION", "v1"); - serviceParams.put("SERVICE_NAME", checkIfMandatory(component.getName(), "name", entity)); - + serviceParams.put("ARTIFACT_DESCRIPTOR_VERSION", new Parameter(checkIfMandatory(bomContent.getMetadata().getComponent().getVersion(), "version in metadata", entity), SBOM_ORIGIN, false)); + serviceParams.put("DEPLOYMENT_RESOURCE_NAME", new Parameter(checkIfMandatory(component.getName(), "name", entity) + "-v1", ENVGENE_CALCULATED, false)); + serviceParams.put("DEPLOYMENT_VERSION", new Parameter("v1", ENVGENE_DEFAULT, false)); + serviceParams.put("SERVICE_NAME", new Parameter(checkIfMandatory(component.getName(), "name", entity), SBOM_ORIGIN, false)); if (CollectionUtils.isNotEmpty(component.getComponents())) { for (Component subComponent : component.getComponents()) { if (subComponent.getMimeType().equalsIgnoreCase("application/vnd.qubership.resource-profile-baseline")) { @@ -304,14 +339,18 @@ private void processImageServiceComponent(EntitiesMap entitiesMap, Component com Map serviceParams = new TreeMap<>(); String tag = null; String entity = "service:" + component.getName(); - serviceParams.put("ARTIFACT_DESCRIPTOR_VERSION", checkIfMandatory(bomContent.getMetadata().getComponent().getVersion(), "version in metadata", entity)); - serviceParams.put("DEPLOYMENT_RESOURCE_NAME", checkIfMandatory(component.getName(), "name", entity) + "-v1"); - serviceParams.put("DEPLOYMENT_VERSION", "v1"); - serviceParams.put("SERVICE_NAME", checkIfMandatory(component.getName(), "name", entity)); String dockerTag = getPropertyValue(component, "full_image_name", null, true, entity); - serviceParams.put("DOCKER_TAG", dockerTag); - serviceParams.put("IMAGE_REPOSITORY", getImageRepository(dockerTag)); - addImageParameters(component, entitiesMap.getDeployParams(), entitiesMap.getServiceMap().keySet()); + Object imageRepository = getImageRepository(dockerTag); + + serviceParams.put("ARTIFACT_DESCRIPTOR_VERSION", new Parameter(checkIfMandatory(bomContent.getMetadata().getComponent().getVersion(), "version in metadata", entity), SBOM_ORIGIN, false)); + serviceParams.put("DEPLOYMENT_RESOURCE_NAME", new Parameter(checkIfMandatory(component.getName(), "name", entity) + "-v1", ENVGENE_CALCULATED, false)); + serviceParams.put("DEPLOYMENT_VERSION", new Parameter("v1", ENVGENE_DEFAULT, false)); + serviceParams.put("SERVICE_NAME", new Parameter(checkIfMandatory(component.getName(), "name", entity), SBOM_ORIGIN, false)); + serviceParams.put("DOCKER_TAG", new Parameter(dockerTag, SBOM_ORIGIN, false)); + if (imageRepository != null) { + serviceParams.put("IMAGE_REPOSITORY", new Parameter(imageRepository, SBOM_ORIGIN, false)); + } + addImageParameters(component, entitiesMap.getDeployParams(),entitiesMap.getServiceMap().keySet()); if (CollectionUtils.isNotEmpty(component.getComponents())) { for (Component subComponent : component.getComponents()) { @@ -321,7 +360,7 @@ private void processImageServiceComponent(EntitiesMap entitiesMap, Component com profileValues = extractProfileValues(subComponent, appName, component.getName(), override, baseline); } } - serviceParams.put("TAG", checkIfMandatory(tag, "TAG", entity)); + serviceParams.put("TAG", new Parameter(checkIfMandatory(tag, "TAG", entity), SBOM_ORIGIN, false)); } if (MapUtils.isNotEmpty(profileValues)) { serviceParams.putAll(profileValues); @@ -329,12 +368,12 @@ private void processImageServiceComponent(EntitiesMap entitiesMap, Component com perServiceMap.put(component.getName(), serviceParams); } - private void addImageParameters(Component component, Map serviceParams, Set serviceNames) { + private void addImageParameters(Component component, Map serviceParams, Set serviceNames) { if (component.getMimeType().equalsIgnoreCase(APPLICATION_OCTET_STREAM)) { String key = getPropertyValue(component, "deploy_param", null, false, component.getName()); if (StringUtils.isNotEmpty(key) && !serviceNames.contains(key)) { String value = getPropertyValue(component, "full_image_name", null, false, component.getName()); - serviceParams.put(key, value); + serviceParams.put(key, new Parameter(value, SBOM_ORIGIN, false)); } } } @@ -354,11 +393,14 @@ private Map extractProfileValues(Component dataComponent, String } for (ComponentData data : dataComponent.getData()) { if (baseline != null && baseline.equals(data.getName().split("\\.")[0])) { + String baselineOrigin = String.format(RP_BASELINE_ORIGIN, baseline); Content content = data.getContents(); String encodedText = content.getAttachment().getText(); profileValues = fileDataConverter.decodeAndParse(encodedText, new TypeReference>() { }); - + if (MapUtils.isNotEmpty(profileValues)) { + wrapPlainMapWithOrigin(profileValues, baselineOrigin); + } profileService.setOverrideProfiles(appName, serviceName, overrideProfile, profileValues); break; } diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/AdaptiveYamlPython.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/AdaptiveYamlPython.java new file mode 100644 index 000000000..c0e4caddc --- /dev/null +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/AdaptiveYamlPython.java @@ -0,0 +1,165 @@ +/* + * Copyright 2024-2025 NetCracker Technology Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.qubership.cloud.devops.cli.utils.yaml; + +import org.qubership.cloud.devops.commons.utils.Parameter; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Mirrors the Python {@code Decoder} / {@code should_expand} logic (aligned with go-yaml decode counting) + * to decide whether YAML output should expand (dereference) aliases. + * + * @see go-yaml decode + */ +public final class AdaptiveYamlPython { + + /** + * ~500kb of dense object declarations, or ~5kb with 10000% alias expansion (Python comment). + */ + private static final int ALIAS_RATIO_RANGE_LOW = 400_000; + /** + * ~5MB of dense object declarations, or ~4.5MB with 10% alias expansion (Python comment). + */ + private static final int ALIAS_RATIO_RANGE_HIGH = 4_000_000; + /** + * Scale {@link #allowedAliasRatio(int)} between low and high decode counts (subtraction, same as historical Java). + */ + private static final double ALIAS_RATIO_RANGE = + (double) ALIAS_RATIO_RANGE_HIGH - ALIAS_RATIO_RANGE_LOW; + + private AdaptiveYamlPython() { + } + + /** + * @return {@code true} if anchors/aliases should be expanded (alias budget exceeded). + */ + public static boolean shouldExpand(Object data) { + Decoder decoder = new Decoder(); + try { + decoder.unmarshal(data); + } catch (ExcessiveAliasingException e) { + return true; + } + return false; + } + + static boolean isLimitExceeded(int alias, int decode) { + if (decode <= 0) { + return false; + } + return alias > 100 + && decode > 1000 + && ((double) alias / decode) > allowedAliasRatio(decode); + } + + static double allowedAliasRatio(int count) { + if (count <= ALIAS_RATIO_RANGE_LOW) { + return 0.99; + } + if (count >= ALIAS_RATIO_RANGE_HIGH) { + return 0.1; + } + return 0.99 - 0.89 * (count - ALIAS_RATIO_RANGE_LOW) / ALIAS_RATIO_RANGE; + } + + private static final class Statistic { + int complexity = 1; + } + + static final class ExcessiveAliasingException extends RuntimeException { + private static final long serialVersionUID = 1L; + } + + /** + * Replicates go-yaml-style decode/alias mass accounting (same structure as the Python {@code Decoder}). + * Uses {@link HashMap} for seen collections, keyed by {@link Map#equals(Object)} / {@link List#equals(Object)} + * the same way as {@link YamlFileWriter}'s ref-count map. + */ + private static final class Decoder { + private final HashMap unique = new HashMap<>(); + private int decodeCount; + private int aliasCount; + + int unmarshal(Object node) { + Object n = unwrapParameter(node); + if (n == null) { + decodeCount++; + return 1; + } + if (!(n instanceof Map || n instanceof List)) { + decodeCount++; + return 1; + } + + if (unique.containsKey(n)) { + return alias(n); + } + + decodeCount++; + Statistic stat = new Statistic(); + unique.put(n, stat); + + int childComplexity = 0; + if (n instanceof Map map) { + childComplexity += mapping(map); + } else { + childComplexity += sequence((List) n); + } + + stat.complexity += childComplexity; + + if (isLimitExceeded(aliasCount, decodeCount)) { + throw new ExcessiveAliasingException(); + } + return stat.complexity; + } + + private int alias(Object n) { + Statistic stat = unique.get(n); + decodeCount += stat.complexity; + aliasCount += stat.complexity; + return stat.complexity; + } + + private static Object unwrapParameter(Object node) { + if (node instanceof Parameter p) { + return p.getValue(); + } + return node; + } + + private int mapping(Map node) { + int complexity = 0; + for (Map.Entry e : node.entrySet()) { + complexity += unmarshal(e.getKey()); + complexity += unmarshal(e.getValue()); + } + return complexity; + } + + private int sequence(List node) { + int complexity = 0; + for (Object item : node) { + complexity += unmarshal(item); + } + return complexity; + } + } +} diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/YamlAnchorRegistry.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/YamlAnchorRegistry.java new file mode 100644 index 000000000..9f3164a0f --- /dev/null +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/YamlAnchorRegistry.java @@ -0,0 +1,64 @@ +/* + * Copyright 2024-2025 NetCracker Technology Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.qubership.cloud.devops.cli.utils.yaml; + +import org.snakeyaml.engine.v2.common.Anchor; +import org.snakeyaml.engine.v2.nodes.AnchorNode; +import org.snakeyaml.engine.v2.nodes.Node; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; + +public class YamlAnchorRegistry { + + private final Map refCount; + private final Map builtNodes = new HashMap<>(); + private final AtomicInteger nextAnchorId = new AtomicInteger(1); + + public YamlAnchorRegistry(Map refCount) { + this.refCount = refCount; + } + + public Node getAliasNodeIfExists(Object obj) { + if (!shouldAnchor(obj) || !builtNodes.containsKey(obj)) { + return null; + } + return new AnchorNode(builtNodes.get(obj)); + } + + public void registerAnchorIfNeeded(Object obj, Node node) { + if (!shouldAnchor(obj)) { + return; + } + builtNodes.put(obj, node); + node.setAnchor(Optional.of(new Anchor(nextAnchorId()))); + } + + private boolean shouldAnchor(Object obj) { + if ((obj instanceof Collection c && c.isEmpty()) || (obj instanceof Map m && m.isEmpty())) { + return false; + } + return refCount.getOrDefault(obj, 0) > 1; + } + + private String nextAnchorId() { + return String.format("id%03d", nextAnchorId.getAndIncrement()); + } +} diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/YamlFileWriter.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/YamlFileWriter.java new file mode 100644 index 000000000..a1b6913e7 --- /dev/null +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/YamlFileWriter.java @@ -0,0 +1,96 @@ +/* + * Copyright 2024-2025 NetCracker Technology Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.qubership.cloud.devops.cli.utils.yaml; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import lombok.extern.slf4j.Slf4j; +import org.qubership.cloud.devops.cli.pojo.dto.shared.SharedData; +import org.qubership.cloud.devops.cli.utils.FileSystemUtils; +import org.snakeyaml.engine.v2.api.Dump; +import org.snakeyaml.engine.v2.api.DumpSettings; +import org.snakeyaml.engine.v2.api.StreamDataWriter; +import org.snakeyaml.engine.v2.common.FlowStyle; +import org.snakeyaml.engine.v2.nodes.Node; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.StringWriter; +import java.util.Map; +import java.util.Set; + +@ApplicationScoped +@Slf4j +public class YamlFileWriter { + + private static final Set EXCLUDED_FILES = Set.of("mapping.yaml"); + private static final String DEPLOY_DESCRIPTOR_FILE_NAME = "deploy-descriptor.yaml"; + private static final String SBOM_HEADER = "#Source of parameters not marked inline: `#sbom`\n"; + + private final FileSystemUtils fileSystemUtils; + private final SharedData sharedData; + private final YamlNodeBuilder yamlNodeBuilder; + + @Inject + public YamlFileWriter(FileSystemUtils fileSystemUtils, SharedData sharedData, YamlNodeBuilder yamlNodeBuilder) { + this.fileSystemUtils = fileSystemUtils; + this.sharedData = sharedData; + this.yamlNodeBuilder = yamlNodeBuilder; + } + + public void write(Map params, String... path) throws IOException { + File file = fileSystemUtils.getFileFromGivenPath(path); + + boolean enableTraceability = + sharedData != null + && sharedData.isEnableTraceability() + && !EXCLUDED_FILES.contains(file.getName()); + boolean isDeployDescriptor = DEPLOY_DESCRIPTOR_FILE_NAME.equals(file.getName()); + + try (BufferedWriter writer = new BufferedWriter(new FileWriter(file))) { + if (params == null || params.isEmpty()) { + return; + } + + if (isDeployDescriptor && enableTraceability) { + writer.write(SBOM_HEADER); + } + + writer.write(dump(params, enableTraceability, isDeployDescriptor)); + } + } + + public String dump(Map data, boolean enableTraceability, boolean deployDescriptorYaml) { + Node root = yamlNodeBuilder.build(data, enableTraceability, deployDescriptorYaml); + + DumpSettings settings = DumpSettings.builder() + .setDumpComments(true) + .setDefaultFlowStyle(FlowStyle.BLOCK) + .setDereferenceAliases(AdaptiveYamlPython.shouldExpand(data)) + .build(); + + Dump dump = new Dump(settings); + StringStreamWriter writer = new StringStreamWriter(); + dump.dumpNode(root, writer); + return writer.toString(); + } + + private static final class StringStreamWriter extends StringWriter implements StreamDataWriter { + } +} diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/YamlNodeBuilder.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/YamlNodeBuilder.java new file mode 100644 index 000000000..edbf95060 --- /dev/null +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/YamlNodeBuilder.java @@ -0,0 +1,194 @@ +/* + * Copyright 2024-2025 NetCracker Technology Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.qubership.cloud.devops.cli.utils.yaml; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.qubership.cloud.devops.commons.utils.Parameter; +import org.snakeyaml.engine.v2.comments.CommentType; +import org.snakeyaml.engine.v2.common.FlowStyle; +import org.snakeyaml.engine.v2.common.ScalarStyle; +import org.snakeyaml.engine.v2.nodes.MappingNode; +import org.snakeyaml.engine.v2.nodes.Node; +import org.snakeyaml.engine.v2.nodes.NodeTuple; +import org.snakeyaml.engine.v2.nodes.ScalarNode; +import org.snakeyaml.engine.v2.nodes.SequenceNode; +import org.snakeyaml.engine.v2.nodes.Tag; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +@ApplicationScoped +public class YamlNodeBuilder { + private final YamlNodeCommentHelper commentHelper; + private final YamlReferenceCounter referenceCounter; + + @Inject + public YamlNodeBuilder(YamlNodeCommentHelper commentHelper, YamlReferenceCounter referenceCounter) { + this.commentHelper = commentHelper; + this.referenceCounter = referenceCounter; + } + + public Node build(Object input, boolean enableTraceability, boolean deployDescriptorYaml) { + YamlAnchorRegistry anchorRegistry = new YamlAnchorRegistry(referenceCounter.countReferences(input)); + return build(input, enableTraceability, deployDescriptorYaml, anchorRegistry); + } + + private Node build(Object input, boolean enableTraceability, boolean deployDescriptorYaml, + YamlAnchorRegistry anchorRegistry) { + + String origin = null; + Object value = input; + + if (input instanceof Parameter p) { + origin = p.getOrigin(); + value = p.getValue(); + } + + boolean addComment = commentHelper.shouldAddComment(enableTraceability, origin, deployDescriptorYaml); + + if (value == null) { + return scalarNode("null", Tag.NULL, origin, addComment); + } + + if (value instanceof String str) { + return handleString(str, origin, addComment); + } + + if (value instanceof Number num) { + return handleNumber(num, origin, addComment); + } + + if (value instanceof Boolean bool) { + return scalarNode(bool.toString(), Tag.BOOL, origin, addComment); + } + + if (value instanceof List list) { + return handleList(list, origin, addComment, enableTraceability, deployDescriptorYaml, anchorRegistry); + } + + if (value instanceof Map map) { + return handleMap(map, origin, addComment, enableTraceability, deployDescriptorYaml, anchorRegistry); + } + + return scalarNode(value.toString(), Tag.STR, origin, addComment); + } + + private Node handleString(String str, String origin, boolean addComment) { + if ("!merge".equals(str)) { + return new ScalarNode(Tag.STR, "<<", ScalarStyle.PLAIN); + } + boolean multiline = str.contains("\n"); + boolean looksLikeBoolean = "True".equals(str) || "False".equals(str); + ScalarStyle scalarStyle; + CommentType commentType; + + if (multiline) { + scalarStyle = ScalarStyle.LITERAL; + commentType = CommentType.BLOCK; + } else if (looksLikeBoolean) { + scalarStyle = ScalarStyle.SINGLE_QUOTED; + commentType = CommentType.IN_LINE; + } else { + scalarStyle = ScalarStyle.PLAIN; + commentType = CommentType.IN_LINE; + } + return commentHelper.attachComment(new ScalarNode(Tag.STR, str, scalarStyle), origin, addComment, commentType); + } + + private Node handleNumber(Number num, String origin, boolean addComment) { + Tag tag = (num instanceof Float || num instanceof Double) ? Tag.FLOAT : Tag.INT; + return scalarNode(num.toString(), tag, origin, addComment); + } + + private Node handleList(List list, String origin, boolean addComment, boolean enableTraceability, + boolean deployDescriptorYaml, YamlAnchorRegistry anchorRegistry) { + + if (list.isEmpty()) { + return createEmptyListNode(origin, addComment); + } + + return buildSequenceNode(list, origin, addComment, enableTraceability, deployDescriptorYaml, anchorRegistry); + } + + private Node handleMap(Map map, String origin, boolean addComment, boolean enableTraceability, + boolean deployDescriptorYaml, YamlAnchorRegistry anchorRegistry) { + + if (map.isEmpty()) { + return createEmptyMapNode(origin, addComment); + } + + return buildMappingNode(map, origin, addComment, enableTraceability, deployDescriptorYaml, anchorRegistry); + } + + private Node scalarNode(String value, Tag tag, String origin, boolean addComment) { + return commentHelper.attachComment(new ScalarNode(tag, value, ScalarStyle.PLAIN), origin, addComment, CommentType.IN_LINE); + } + + private Node buildSequenceNode(List list, String origin, boolean addComment, boolean enableTraceability, + boolean deployDescriptorYaml, YamlAnchorRegistry anchorRegistry) { + Node aliasNode = anchorRegistry.getAliasNodeIfExists(list); + if (aliasNode != null) { + return aliasNode; + } + + List children = new ArrayList<>(); + SequenceNode seqNode = new SequenceNode(Tag.SEQ, children, FlowStyle.BLOCK); + + anchorRegistry.registerAnchorIfNeeded(list, seqNode); + + for (Object elem : list) { + children.add(this.build(elem, enableTraceability, deployDescriptorYaml, anchorRegistry)); + } + + return commentHelper.attachComment(seqNode, origin, addComment, CommentType.BLOCK); + } + + private Node buildMappingNode(Map map, String origin, boolean addComment, boolean enableTraceability, + boolean deployDescriptorYaml, YamlAnchorRegistry anchorRegistry) { + Node aliasNode = anchorRegistry.getAliasNodeIfExists(map); + if (aliasNode != null) { + return aliasNode; + } + + List tuples = new ArrayList<>(); + MappingNode mapping = new MappingNode(Tag.MAP, tuples, FlowStyle.BLOCK); + + anchorRegistry.registerAnchorIfNeeded(map, mapping); + + for (Map.Entry entry : map.entrySet()) { + Node keyNode = this.build(entry.getKey(), enableTraceability, deployDescriptorYaml, anchorRegistry); + Node valueNode = this.build(entry.getValue(), enableTraceability, deployDescriptorYaml, anchorRegistry); + commentHelper.moveBlockCommentFromValueToKey(valueNode, keyNode); + tuples.add(new NodeTuple(keyNode, valueNode)); + } + + return commentHelper.attachComment(mapping, origin, addComment, CommentType.BLOCK); + } + + private Node createEmptyMapNode(String origin, boolean addComment) { + MappingNode node = new MappingNode(Tag.MAP, Collections.emptyList(), FlowStyle.FLOW); + return commentHelper.attachComment(node, origin, addComment, CommentType.IN_LINE); + } + + private Node createEmptyListNode(String origin, boolean addComment) { + SequenceNode node = new SequenceNode(Tag.SEQ, Collections.emptyList(), FlowStyle.FLOW); + return commentHelper.attachComment(node, origin, addComment, CommentType.IN_LINE); + } +} diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/YamlNodeCommentHelper.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/YamlNodeCommentHelper.java new file mode 100644 index 000000000..a320dbfb3 --- /dev/null +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/YamlNodeCommentHelper.java @@ -0,0 +1,63 @@ +/* + * Copyright 2024-2025 NetCracker Technology Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.qubership.cloud.devops.cli.utils.yaml; + +import jakarta.enterprise.context.ApplicationScoped; +import org.apache.commons.lang3.StringUtils; +import org.snakeyaml.engine.v2.comments.CommentLine; +import org.snakeyaml.engine.v2.comments.CommentType; +import org.snakeyaml.engine.v2.nodes.Node; + +import java.util.List; +import java.util.Locale; +import java.util.Optional; + +import static org.qubership.cloud.devops.commons.utils.constant.ParametersConstants.SBOM_ORIGIN; + +@ApplicationScoped +public class YamlNodeCommentHelper { + + public boolean shouldAddComment(boolean enableTraceability, String origin, boolean deployDescriptorYaml) { + if (!enableTraceability || StringUtils.isBlank(origin)) { + return false; + } + return !deployDescriptorYaml || !origin.toLowerCase(Locale.ROOT).contains(SBOM_ORIGIN); + } + + public void moveBlockCommentFromValueToKey(Node valueNode, Node keyNode) { + List valueComments = valueNode.getBlockComments(); + if (valueComments == null || valueComments.isEmpty()) { + return; + } + keyNode.setBlockComments(valueComments); + valueNode.setBlockComments(null); + } + + public T attachComment(T node, String origin, boolean addComment, CommentType commentType) { + if (!addComment) { + return node; + } + CommentLine comment = new CommentLine(Optional.empty(), Optional.empty(), origin, commentType); + List comments = List.of(comment); + if (commentType == CommentType.BLOCK) { + node.setBlockComments(comments); + } else { + node.setInLineComments(comments); + } + return node; + } +} diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/YamlReferenceCounter.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/YamlReferenceCounter.java new file mode 100644 index 000000000..9984bee07 --- /dev/null +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/YamlReferenceCounter.java @@ -0,0 +1,63 @@ +/* + * Copyright 2024-2025 NetCracker Technology Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.qubership.cloud.devops.cli.utils.yaml; + +import jakarta.enterprise.context.ApplicationScoped; +import org.qubership.cloud.devops.commons.utils.Parameter; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@ApplicationScoped +public class YamlReferenceCounter { + + public Map countReferences(Object root) { + Map refCount = new HashMap<>(); + countReferencesRecursively(root, refCount); + return refCount; + } + + private void countReferencesRecursively(Object obj, Map refCount) { + if (obj == null) { + return; + } + + Object value = (obj instanceof Parameter p) ? p.getValue() : obj; + if (value instanceof Map map) { + if (incrementAndCheckVisited(map, refCount)) { + return; + } + for (Object v : map.values()) { + countReferencesRecursively(v, refCount); + } + } else if (value instanceof List list) { + if (incrementAndCheckVisited(list, refCount)) { + return; + } + for (Object item : list) { + countReferencesRecursively(item, refCount); + } + } + } + + private boolean incrementAndCheckVisited(Object obj, Map refCount) { + int count = refCount.getOrDefault(obj, 0); + refCount.put(obj, count + 1); + return count > 0; + } +} diff --git a/build_effective_set_generator/effective-set-generator/src/test/java/org/qubership/cloud/devops/cli/CmdbCliTest.java b/build_effective_set_generator/effective-set-generator/src/test/java/org/qubership/cloud/devops/cli/CmdbCliTest.java index 56fe74f12..dd126b7b7 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/java/org/qubership/cloud/devops/cli/CmdbCliTest.java +++ b/build_effective_set_generator/effective-set-generator/src/test/java/org/qubership/cloud/devops/cli/CmdbCliTest.java @@ -43,7 +43,6 @@ void testGenerateEffectiveSet(@TempDir Path tempDir) throws Exception { "--app_chart_validation", "false", "--custom-params", "@config.json" ); - assertEquals(0, exitCode); Path expected = FileTestUtils.resource("environments/cluster-01/pl-01/effective-set"); diff --git a/build_effective_set_generator/effective-set-generator/src/test/java/org/qubership/cloud/devops/cli/utils/yaml/AdaptiveYamlPythonTest.java b/build_effective_set_generator/effective-set-generator/src/test/java/org/qubership/cloud/devops/cli/utils/yaml/AdaptiveYamlPythonTest.java new file mode 100644 index 000000000..f79080d97 --- /dev/null +++ b/build_effective_set_generator/effective-set-generator/src/test/java/org/qubership/cloud/devops/cli/utils/yaml/AdaptiveYamlPythonTest.java @@ -0,0 +1,118 @@ +/* + * Copyright 2024-2025 NetCracker Technology Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.qubership.cloud.devops.cli.utils.yaml; + +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.qubership.cloud.devops.cli.pojo.dto.shared.SharedData; +import org.qubership.cloud.devops.cli.utils.FileSystemUtils; +import org.qubership.cloud.devops.commons.utils.Parameter; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class AdaptiveYamlPythonTest { + + @Test + void shouldExpand_emptyMap_returnsFalse() { + assertFalse(AdaptiveYamlPython.shouldExpand(new LinkedHashMap<>())); + } + + @Test + void shouldExpand_smallTreeNoSharing_returnsFalse() { + Map root = new LinkedHashMap<>(); + root.put("a", Map.of("k", "v")); + assertFalse(AdaptiveYamlPython.shouldExpand(root)); + } + + @Test + void shouldExpand_manyAliasesToSameSubtree_returnsTrue() { + Map shared = new LinkedHashMap<>(); + shared.put("payload", "x"); + + // List of repeated refs: few primitives vs many aliases keeps alias/decode above the ratio threshold. + // A flat map of 2000 string keys dilutes decode_count and no longer trips the limit (Python behaves the same). + List manyRefs = new ArrayList<>(2000); + for (int i = 0; i < 2000; i++) { + manyRefs.add(shared); + } + Map root = new LinkedHashMap<>(); + root.put("refs", manyRefs); + assertTrue(AdaptiveYamlPython.shouldExpand(root)); + } + + @Test + void shouldExpand_unwrapsParameterLikeYamlTraversal() { + Map shared = new LinkedHashMap<>(); + shared.put("payload", "x"); + + List manyRefs = new ArrayList<>(2000); + for (int i = 0; i < 2000; i++) { + manyRefs.add(shared); + } + Map inner = new LinkedHashMap<>(); + inner.put("refs", manyRefs); + Map root = new LinkedHashMap<>(); + root.put("wrapped", new Parameter(inner, "origin", false)); + assertTrue(AdaptiveYamlPython.shouldExpand(root)); + } + + @Test + void isLimitExceeded_respectsThresholds() { + assertFalse(AdaptiveYamlPython.isLimitExceeded(50, 2000)); + assertFalse(AdaptiveYamlPython.isLimitExceeded(200, 500)); + assertTrue(AdaptiveYamlPython.isLimitExceeded(1999, 2000)); + } + + @Test + void yamlFileWriter_dump_expandsWhenDecoderExceedsLimit() throws Exception { + Path tempDir = Files.createTempDirectory("yaml-deref"); + File outputFile = tempDir.resolve("out.yaml").toFile(); + + FileSystemUtils fs = Mockito.mock(FileSystemUtils.class); + Mockito.when(fs.getFileFromGivenPath(Mockito.any())).thenReturn(outputFile); + + SharedData sharedData = Mockito.mock(SharedData.class); + Mockito.when(sharedData.isEnableTraceability()).thenReturn(false); + + Map shared = new LinkedHashMap<>(); + shared.put("payload", "x"); + + List manyRefs = new ArrayList<>(2000); + for (int i = 0; i < 2000; i++) { + manyRefs.add(shared); + } + Map root = new LinkedHashMap<>(); + root.put("refs", manyRefs); + + YamlFileWriter yamlFileWriter = new YamlFileWriter( + fs, sharedData, new YamlNodeBuilder(new YamlNodeCommentHelper(), new YamlReferenceCounter())); + yamlFileWriter.write(root, "ignored"); + + String content = Files.readString(outputFile.toPath()); + assertFalse(content.contains("&id"), "Expected dereferenced dump without YAML anchor definitions"); + assertTrue(content.length() > 10_000, "Expanded structure should produce a large document"); + } +} diff --git a/build_effective_set_generator/effective-set-generator/src/test/java/org/qubership/cloud/devops/cli/utils/yaml/YamlFileWriterTest.java b/build_effective_set_generator/effective-set-generator/src/test/java/org/qubership/cloud/devops/cli/utils/yaml/YamlFileWriterTest.java new file mode 100644 index 000000000..73829c750 --- /dev/null +++ b/build_effective_set_generator/effective-set-generator/src/test/java/org/qubership/cloud/devops/cli/utils/yaml/YamlFileWriterTest.java @@ -0,0 +1,207 @@ +package org.qubership.cloud.devops.cli.utils.yaml; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; +import org.qubership.cloud.devops.cli.pojo.dto.shared.SharedData; +import org.qubership.cloud.devops.cli.utils.FileSystemUtils; +import org.qubership.cloud.devops.commons.utils.Parameter; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class YamlFileWriterTest { + + @Test + void shouldWriteYamlToFileWithTraceabilityComments(@TempDir Path tempDir) throws Exception { + File outputFile = tempDir.resolve("traceable-yaml-test.yaml").toFile(); + + FileSystemUtils fs = Mockito.mock(FileSystemUtils.class); + Mockito.when(fs.getFileFromGivenPath(Mockito.any())).thenReturn(outputFile); + + SharedData sharedData = Mockito.mock(SharedData.class); + Mockito.when(sharedData.isEnableTraceability()).thenReturn(true); + + YamlFileWriter yamlFileWriter = new YamlFileWriter( + fs, sharedData, new YamlNodeBuilder(new YamlNodeCommentHelper(), new YamlReferenceCounter())); + Map data = buildFullCombinationTestData(); + + yamlFileWriter.write(data, "ignored"); + + assertTrue(outputFile.exists()); + + String content = Files.readString(outputFile.toPath()); + assertNotNull(content); + // With traceability on, origin comments should appear for Parameters with origin + assertTrue(content.contains("value1") && content.contains("system"), + "Expected inline comment for paramString"); + assertTrue(content.contains("line1") && content.contains("line2"), + "Expected multiline content"); + } + + @Test + void deployDescriptorYaml_hasFileHeaderAndSkipsCommentsOnlyForSbomOrigins(@TempDir Path tempDir) + throws Exception { + File outputFile = tempDir.resolve("deploy-descriptor.yaml").toFile(); + + FileSystemUtils fs = Mockito.mock(FileSystemUtils.class); + Mockito.when(fs.getFileFromGivenPath(Mockito.any())).thenReturn(outputFile); + + SharedData sharedData = Mockito.mock(SharedData.class); + Mockito.when(sharedData.isEnableTraceability()).thenReturn(true); + + YamlFileWriter yamlFileWriter = new YamlFileWriter( + fs, sharedData, new YamlNodeBuilder(new YamlNodeCommentHelper(), new YamlReferenceCounter())); + Map root = new LinkedHashMap<>(); + root.put( + "fromSbom", + new Parameter("digest-val", "/repo/app-sbom.json:docker_digest", false)); + root.put("fromBaseline", new Parameter("500m", "rp-baseline: dev", false)); + + yamlFileWriter.write(root, "ignored"); + + String content = Files.readString(outputFile.toPath()); + assertTrue(content.startsWith("#Source of parameters not marked inline: `#sbom`")); + assertTrue(content.contains("digest-val")); + assertFalse(content.contains("app-sbom.json")); + assertTrue(content.contains("500m")); + assertTrue(content.contains("rp-baseline")); + } + + @Test + void shouldWriteYamlToFileWithoutTraceability(@TempDir Path tempDir) throws Exception { + File outputFile = tempDir.resolve("traceable-yaml-no-trace.yaml").toFile(); + + FileSystemUtils fs = Mockito.mock(FileSystemUtils.class); + Mockito.when(fs.getFileFromGivenPath(Mockito.any())).thenReturn(outputFile); + + SharedData sharedData = Mockito.mock(SharedData.class); + Mockito.when(sharedData.isEnableTraceability()).thenReturn(false); + + YamlFileWriter yamlFileWriter = new YamlFileWriter( + fs, sharedData, new YamlNodeBuilder(new YamlNodeCommentHelper(), new YamlReferenceCounter())); + Map data = buildFullCombinationTestData(); + + yamlFileWriter.write(data, "ignored"); + + assertTrue(outputFile.exists()); + String content = Files.readString(outputFile.toPath()); + assertNotNull(content); + assertTrue(content.contains("value1")); + } + + private Map buildFullCombinationTestData() { + + Map root = new LinkedHashMap<>(); + + + root.put("stringValue", "plainText"); + root.put("intValue", 100); + root.put("booleanValue", true); + root.put("nullValue", null); + + root.put("paramString", + new Parameter("value1", "system",false)); + + root.put("paramNumber", + new Parameter(200, "default",false)); + + + root.put("paramBlankOrigin", + new Parameter("noComment", "",false)); + + + root.put("paramNullOrigin", + new Parameter("noComment2", null,false)); + + + root.put("multiLineParam", + new Parameter("line1\nline2\nline3", "file",false)); + + + root.put("simpleList", List.of("a", "b", "c")); + + root.put("emptyList", new ArrayList<>()); + + + root.put("paramList", List.of( + new Parameter("srv1", "inventory",false), + new Parameter("srv2", "inventory",false) + )); + + + root.put("mixedList", List.of( + 8080, + new Parameter(9090, "config",false), + "text", + new Parameter("text2", "runtime",false) + )); + + + root.put("listOfMaps", List.of( + Map.of( + "name", new Parameter("dev", "profile",false), + "enabled", true + ), + Map.of( + "name", new Parameter("prod", "profile",false), + "enabled", false + ) + )); + + root.put("nestedMap", Map.of( + "level1", Map.of( + "level2", new Parameter("deepValue", "deepOrigin",false) + ) + )); + + root.put("mapOfLists", Map.of( + "numbers", List.of( + new Parameter(1, "src1",false), + new Parameter(2, "src2",false), + new Parameter(3, "src3",false), + 4, + 5 + ), + "strings", List.of( + "x", + new Parameter("y\nz", "src",false) + ) + )); + + root.put("paramWrappingMap", + new Parameter( + Map.of( + "innerKey", + new Parameter("innerValue", "innerOrigin",false) + ), + "outerOrigin",false + ) + ); + + root.put("paramWrappingList", + new Parameter( + List.of( + new Parameter("inside1", "listOrigin",false), + "plainInside" + ), + "outerListOrigin",false + ) + ); + + + root.put("emptyMap", new LinkedHashMap<>()); + + + root.put("mergeExample", "!merge"); + + return root; + } +} diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/credentials.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/credentials.yaml index 0dfe41d91..b28696244 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/credentials.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/credentials.yaml @@ -6,6 +6,7 @@ K8S_TOKEN: token-placeholder-123 KMS_CERT_IN_BASE64: token-placeholder-123 STORAGE_PASSWORD: pass-placeholder-123 STORAGE_USERNAME: user-placeholder-123 +TEST_CMDB_CREDS_USERNAME: user-placeholder-123 TEST_CREDS_GET_PASSWORD_PASSWORD: pass-placeholder-123 TEST_CREDS_GET_PASSWORD_USERNAME: user-placeholder-123 TEST_CREDS_GET_SECRET_PARAM: token-placeholder-123 @@ -21,7 +22,6 @@ TEST_ENVGENE_CREDS_GET_VAULT_ROLE: envgeneNullValue TEST_ENVGENE_CREDS_GET_VAULT_SECRET_ID: envgeneNullValue TEST_SHARED_CREDS: user-placeholder-123 TEST_SHARED_CREDS_ACTIVATOR: pass-placeholder-123 -TEST_CMDB_CREDS_USERNAME: user-placeholder-123 graphite-remote-adapter: user-placeholder-123 kafka: password: pass-placeholder-123 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/credentials.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/credentials.yaml index 2f494be22..b4ba9d4d6 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/credentials.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/credentials.yaml @@ -13,6 +13,7 @@ MAAS_CREDENTIALS_PASSWORD: pass-placeholder-123 MAAS_CREDENTIALS_USERNAME: user-placeholder-123 STORAGE_PASSWORD: pass-placeholder-123 STORAGE_USERNAME: user-placeholder-123 +TEST_CMDB_CREDS_USERNAME: user-placeholder-123 TEST_CREDS_GET_PASSWORD_PASSWORD: pass-placeholder-123 TEST_CREDS_GET_PASSWORD_USERNAME: user-placeholder-123 TEST_CREDS_GET_SECRET_PARAM: token-placeholder-123 @@ -28,7 +29,6 @@ TEST_ENVGENE_CREDS_GET_VAULT_ROLE: envgeneNullValue TEST_ENVGENE_CREDS_GET_VAULT_SECRET_ID: envgeneNullValue TEST_SHARED_CREDS: user-placeholder-123 TEST_SHARED_CREDS_ACTIVATOR: pass-placeholder-123 -TEST_CMDB_CREDS_USERNAME: user-placeholder-123 kafka: &id001 password: pass-placeholder-123 username: user-placeholder-123 @@ -48,6 +48,7 @@ global: &id002 MAAS_CREDENTIALS_USERNAME: user-placeholder-123 STORAGE_PASSWORD: pass-placeholder-123 STORAGE_USERNAME: user-placeholder-123 + TEST_CMDB_CREDS_USERNAME: user-placeholder-123 TEST_CREDS_GET_PASSWORD_PASSWORD: pass-placeholder-123 TEST_CREDS_GET_PASSWORD_USERNAME: user-placeholder-123 TEST_CREDS_GET_SECRET_PARAM: token-placeholder-123 @@ -63,7 +64,6 @@ global: &id002 TEST_ENVGENE_CREDS_GET_VAULT_SECRET_ID: envgeneNullValue TEST_SHARED_CREDS: user-placeholder-123 TEST_SHARED_CREDS_ACTIVATOR: pass-placeholder-123 - TEST_CMDB_CREDS_USERNAME: user-placeholder-123 graphite-remote-adapter: user-placeholder-123 kafka: *id001 alertmanager: *id002 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/deploy-descriptor.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/deploy-descriptor.yaml index 52036e226..651782423 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/deploy-descriptor.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/deploy-descriptor.yaml @@ -1,5 +1,5 @@ -global: &id002 - deployDescriptor: &id001 +global: &id001 + deployDescriptor: &id002 alertmanager: artifacts: [] deploy_param: '' @@ -810,13 +810,13 @@ global: &id002 promote_artifacts: '' qualifier: '' version: '' -deployDescriptor: *id001 +deployDescriptor: *id002 alertmanager: &id003 APPLICATION_NAME: MONITORING DEPLOYMENT_SESSION_ID: 6d5a6ce9-0b55-429d-8877-f7a88dae3d9c MANAGED_BY: argocd - deployDescriptor: *id001 - global: *id002 + deployDescriptor: *id002 + global: *id001 blackbox-exporter: *id003 cert-exporter: *id003 cloud-events-exporter: *id003 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deploy-descriptor.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deploy-descriptor.yaml index 775d3ba85..63e3eceee 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deploy-descriptor.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deploy-descriptor.yaml @@ -1,5 +1,5 @@ -global: &id002 - deployDescriptor: &id001 +global: &id001 + deployDescriptor: &id002 patroni-core: artifacts: [] deploy_param: '' @@ -126,13 +126,13 @@ global: &id002 promote_artifacts: '' qualifier: '' version: '' -deployDescriptor: *id001 +deployDescriptor: *id002 patroni-core: &id003 APPLICATION_NAME: postgres DEPLOYMENT_SESSION_ID: 6d5a6ce9-0b55-429d-8877-f7a88dae3d9c MANAGED_BY: argocd - deployDescriptor: *id001 - global: *id002 + deployDescriptor: *id002 + global: *id001 pg_patroni: *id003 pg_upgrade: *id003 pgbackrest_sidecar: *id003 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring-origin/MONITORING/parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring-origin/MONITORING/parameters.yaml index 0e3a1acf8..db95ede9f 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring-origin/MONITORING/parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring-origin/MONITORING/parameters.yaml @@ -1,5 +1,5 @@ PARAM_2: value-2 PARAM_6: value-6 TECHNICAL_PARAM_1: VALUE_TP_1 -integrations.ndo-api-gw.url: "http://api-test:8080" -artifact: "org.snakeyaml" +artifact: org.snakeyaml +integrations.ndo-api-gw.url: http://api-test:8080 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/pg/postgres/parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/pg/postgres/parameters.yaml index 87987d319..5ecdc1906 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/pg/postgres/parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/pg/postgres/parameters.yaml @@ -1,2 +1,2 @@ -integrations.ndo-api-gw.url: "http://api-test:8080" -artifact: "org.snakeyaml" +artifact: org.snakeyaml +integrations.ndo-api-gw.url: http://api-test:8080 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/credentials.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/credentials.yaml index 511db0adf..6f688f4bd 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/credentials.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/credentials.yaml @@ -1,7 +1,7 @@ bg_domain: controllerNamespace: - password: pass-placeholder-123 username: user-placeholder-123 + password: pass-placeholder-123 k8s_tokens: pl-01-pg: token-placeholder-123 pl-01-monitoring: token-placeholder-123 diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/ExpressionLanguage.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/ExpressionLanguage.java index 328b50068..ab05efea5 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/ExpressionLanguage.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/ExpressionLanguage.java @@ -84,14 +84,15 @@ public ExpressionLanguage(Binding binding) { this.gStringToJinJavaTranslator = new GStringToJinJavaTranslator(); this.binding.forEach((key1, value) -> - this.binding.put(key1, translateParameter(value.getValue()))); + this.binding.put(key1, translateParameter(value.getValue(), ""))); + this.dynamicResolver = new DynamicPropertyResolver(this.binding); } - private Parameter translateParameter(Object value) { + private Parameter translateParameter(Object value, String parentOrigin) { Object val = getValue(value); - + String origin = resolveOrigin(value, parentOrigin); if (val instanceof String) { String strVal = (String) val; strVal = strVal.replaceAll("\\\\\"", "\""); @@ -100,41 +101,50 @@ private Parameter translateParameter(Object value) { Parameter oldParameter = (Parameter) value; return new Parameter( strVal, - oldParameter.getOrigin(), + origin, oldParameter.isParsed(), oldParameter.isSecured(), gStringToJinJavaTranslator.translate(strVal)); } else { - return new Parameter( - strVal, - gStringToJinJavaTranslator.translate(strVal)); + return new Parameter(strVal, origin, false, false, gStringToJinJavaTranslator.translate(strVal)); } } else if (val instanceof List) { - return translateList((List) val); + return translateList((List) val, origin); } else if (val instanceof Map) { - return translateMap((Map) val); + return translateMap((Map) val, origin); } if (value instanceof Parameter) { + ((Parameter) value).setOrigin(origin); return (Parameter) value; } - return new Parameter(value); + return new Parameter(value, origin, false); } - private Parameter translateMap(Map map) { - map.replaceAll((k, v) -> translateParameter(v)); + private Parameter translateMap(Map map, String parentOrigin) { + map.replaceAll((k, v) -> translateParameter(v, parentOrigin)); return new Parameter(map); } - private Parameter translateList(List list) { + private Parameter translateList(List list, String parentOrigin) { for (final ListIterator it = list.listIterator(); it.hasNext(); ) { Object element = it.next(); - it.set(translateParameter(element)); + it.set(translateParameter(element, parentOrigin)); } return new Parameter(list); } + private String resolveOrigin(Object value, String parentOrigin) { + if (value instanceof Parameter) { + String origin = ((Parameter) value).getOrigin(); + if (origin != null && !origin.isEmpty()) { + return origin; + } + } + return parentOrigin; + } + private void setUpJinJava() { this.jinjava.registerFilter(new EnvSuffixFilter()); this.jinjava.registerFilter(new EnvPrefixFilter()); diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/ApplicationMap.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/ApplicationMap.java index dec252e9d..f253fe5f2 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/ApplicationMap.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/ApplicationMap.java @@ -38,7 +38,7 @@ public Map getMap(String key) { Application config = Injector.getInstance().getDi().get(ApplicationService.class).getByName(key, namespace); if (config != null) { - Map map = new EscapeMap(config.getParams(), binding, String.format(ParametersConstants.APP_ORIGIN, key)); + Map map = new EscapeMap(config.getParams(), binding, ParametersConstants.APP_ORIGIN); maps.put(key, map); return map; } diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/Binding.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/Binding.java index e7a9160f6..8cfdc2bb0 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/Binding.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/Binding.java @@ -34,6 +34,8 @@ import java.util.*; import java.util.stream.Collectors; +import static org.qubership.cloud.devops.commons.utils.constant.ParametersConstants.APP_ORIGIN; + @Slf4j public class Binding extends HashMap implements Cloneable { @@ -63,13 +65,13 @@ public ParametersParser getParser() { private void processSet(String tenant, String setName, String application, EscapeMap parameterSet, EscapeMap applicationMap) { ParameterSet set = Injector.getInstance().getParameterSetService().getParameterSet(tenant, setName); if (set != null) { - parameterSet.putAllStrings(set.getParameters(), String.format(ParametersConstants.PARAMETER_SET_ORIGIN, setName)); + parameterSet.putAllStrings(set.getParameters(),APP_ORIGIN); ParameterSetApplication parameterSetApplication = set.getApplications().stream() .filter(app -> app.getAppName().equals(application)) .findFirst() .orElse(null); if (parameterSetApplication != null) { - applicationMap.putAllStrings(parameterSetApplication.getParameters(), String.format(ParametersConstants.PARAMETER_SET_APP_ORIGIN, setName, application)); + applicationMap.putAllStrings(parameterSetApplication.getParameters(), APP_ORIGIN); } } } @@ -145,7 +147,7 @@ private Map calculateCredentialsAndPrepareStructuredParams(Ma if (param.getValue() instanceof Map) { return new HashMap() { { - put(entry.getKey(), new Parameter(calculateCredentialsAndPrepareStructuredParams((Map) param.getValue()))); + put(entry.getKey(), new Parameter(calculateCredentialsAndPrepareStructuredParams((Map) param.getValue()), param.getOrigin(), false)); } }; } diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/CloudApplicationMap.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/CloudApplicationMap.java index c870e8925..93c0f6dcd 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/CloudApplicationMap.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/CloudApplicationMap.java @@ -16,7 +16,6 @@ package org.qubership.cloud.parameters.processor.expression.binding; -import org.qubership.cloud.devops.commons.utils.constant.ParametersConstants; import org.qubership.cloud.devops.commons.pojo.applications.model.ApplicationParams; import org.qubership.cloud.devops.commons.pojo.clouds.model.Cloud; import org.qubership.cloud.devops.commons.utils.Parameter; @@ -25,6 +24,9 @@ import java.util.HashMap; import java.util.Map; +import static org.qubership.cloud.devops.commons.utils.constant.ParametersConstants.APP_ORIGIN; +import static org.qubership.cloud.devops.commons.utils.constant.ParametersConstants.CLOUD_ORIGIN; + public class CloudApplicationMap extends DynamicMap { private final Cloud cloud; @@ -43,27 +45,21 @@ public Map getMap(String appName) { Map appParams = applicationParams != null ? applicationParams.getAppParams() : new HashMap<>(); Map configServerParams = applicationParams != null ? applicationParams.getConfigServerParams() : new HashMap<>(); - EscapeMap map = new EscapeMap(appParams, binding, - String.format(ParametersConstants.CLOUD_APP_ORIGIN, cloud.getTenant().getName(), cloud.getName(), appName)); - EscapeMap configServerMap = new EscapeMap(configServerParams, binding, - String.format(ParametersConstants.CLOUD_APP_CONFIG_SERVER_ORIGIN, cloud.getTenant().getName(), cloud.getName(), appName)); + EscapeMap map = new EscapeMap(appParams, binding, APP_ORIGIN); + EscapeMap configServerMap = new EscapeMap(configServerParams, binding, APP_ORIGIN); EscapeMap parameterSetMap = new EscapeMap(null, binding, ""); map.put("parameterSet", parameterSetMap); if (cloud.getDeploymentParameterSets() != null) { new ArrayDeque<>(cloud.getDeploymentParameterSets()).descendingIterator().forEachRemaining(set -> { - String origin = String.format(ParametersConstants.CLOUD_PARAMETER_SET_APP_ORIGIN, - cloud.getTenant().getName(), cloud.getName(), set, appName); - processApplicationSet(cloud.getTenant().getName(), set, appName, origin, parameterSetMap); + processApplicationSet(cloud.getTenant().getName(), set, appName, APP_ORIGIN, parameterSetMap); }); } EscapeMap parameterSetConfigServerMap = new EscapeMap(null, binding, ""); configServerMap.put("parameterSet", parameterSetConfigServerMap); if (cloud.getTechnicalParameterSets() != null) { new ArrayDeque<>(cloud.getTechnicalParameterSets()).descendingIterator().forEachRemaining(set -> { - String origin = String.format(ParametersConstants.CLOUD_PARAMETER_SET_APP_ORIGIN, - cloud.getTenant().getName(), cloud.getName(), set, appName); - processApplicationSet(cloud.getTenant().getName(), set, appName, origin, configServerMap); + processApplicationSet(cloud.getTenant().getName(), set, appName, APP_ORIGIN, configServerMap); }); } diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/CloudMap.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/CloudMap.java index 4334a6e1c..322cdad51 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/CloudMap.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/CloudMap.java @@ -37,6 +37,8 @@ import java.util.Map; import static org.qubership.cloud.devops.commons.utils.constant.CredentialConstants.*; +import static org.qubership.cloud.devops.commons.utils.constant.ParametersConstants.ENVGENE_CALCULATED; +import static org.qubership.cloud.devops.commons.utils.constant.ParametersConstants.ENVGENE_DEFAULT; public class CloudMap extends DynamicMap { @@ -74,9 +76,9 @@ public Map getMap(String cloudName) { return null; // return empty map instead? } mergeE2E = config.isMergeCloudAndE2EParameters(); - EscapeMap map = new EscapeMap(config.getCloudParams(), binding, String.format(ParametersConstants.CLOUD_ORIGIN, tenant, cloudName)); - EscapeMap e2e = new EscapeMap(config.getE2eParams(), binding, String.format(ParametersConstants.CLOUD_E2E_ORIGIN, tenant, cloudName)); - EscapeMap configServer = new EscapeMap(config.getConfigServerParams(), binding, String.format(ParametersConstants.CLOUD_CONFIG_SERVER_ORIGIN, tenant, cloudName)); + EscapeMap map = new EscapeMap(config.getCloudParams(), binding, ParametersConstants.CLOUD_ORIGIN); + EscapeMap e2e = new EscapeMap(config.getE2eParams(), binding, ParametersConstants.CLOUD_ORIGIN); + EscapeMap configServer = new EscapeMap(config.getConfigServerParams(), binding, ParametersConstants.CLOUD_ORIGIN); map.put("app", new Parameter(new CloudApplicationMap(config, defaultApp, binding).init())); @@ -85,12 +87,12 @@ public Map getMap(String cloudName) { if (dbaas.getApiUrl() != null) { map.putIfAbsent("API_DBAAS_ADDRESS", dbaas.getApiUrl()); } else { - map.putIfAbsent("API_DBAAS_ADDRESS", ""); + map.putIfAbsent("API_DBAAS_ADDRESS", DEFAULT_EMPTY_STRING,ENVGENE_DEFAULT); } if (dbaas.getAggregatorUrl() != null) { map.putIfAbsent("DBAAS_AGGREGATOR_ADDRESS", dbaas.getAggregatorUrl()); } else { - map.putIfAbsent("DBAAS_AGGREGATOR_ADDRESS", ""); + map.putIfAbsent("DBAAS_AGGREGATOR_ADDRESS", DEFAULT_EMPTY_STRING,ENVGENE_DEFAULT); } Credential cred = credentialUtils.getCredentialsById(dbaas.getCredId()); @@ -100,17 +102,17 @@ public Map getMap(String cloudName) { map.putIfAbsent("DBAAS_CLUSTER_DBA_CREDENTIALS_USERNAME", ((UsernamePasswordCredentials) cred).getUsername()); map.putIfAbsent("DBAAS_CLUSTER_DBA_CREDENTIALS_PASSWORD", ((UsernamePasswordCredentials) cred).getPassword()); } else { - map.putIfAbsent("DBAAS_AGGREGATOR_USERNAME", DEFAULT_DBAAS_AGGREGATOR_LOGIN); - map.putIfAbsent("DBAAS_AGGREGATOR_PASSWORD", DEFAULT_DBAAS_AGGREGATOR_PASSWORD); - map.putIfAbsent("DBAAS_CLUSTER_DBA_CREDENTIALS_USERNAME", DEFAULT_DBAAS_AGGREGATOR_LOGIN); - map.putIfAbsent("DBAAS_CLUSTER_DBA_CREDENTIALS_PASSWORD", DEFAULT_DBAAS_AGGREGATOR_PASSWORD); + map.putIfAbsent("DBAAS_AGGREGATOR_USERNAME", DEFAULT_DBAAS_AGGREGATOR_LOGIN,ENVGENE_DEFAULT); + map.putIfAbsent("DBAAS_AGGREGATOR_PASSWORD", DEFAULT_DBAAS_AGGREGATOR_PASSWORD,ENVGENE_DEFAULT); + map.putIfAbsent("DBAAS_CLUSTER_DBA_CREDENTIALS_USERNAME", DEFAULT_DBAAS_AGGREGATOR_LOGIN,ENVGENE_DEFAULT); + map.putIfAbsent("DBAAS_CLUSTER_DBA_CREDENTIALS_PASSWORD", DEFAULT_DBAAS_AGGREGATOR_PASSWORD,ENVGENE_DEFAULT); } - map.putIfAbsent("DBAAS_ENABLED", new Parameter(dbaas.isEnable())); + map.putIfAbsent("DBAAS_ENABLED", new Parameter(dbaas.isEnable(), ParametersConstants.CLOUD_ORIGIN, false)); } MaaS maas = config.getMaas(); if (maas != null) { - map.putIfAbsent("MAAS_ENABLED", new Parameter(maas.isEnable())); + map.putIfAbsent("MAAS_ENABLED", new Parameter(maas.isEnable(), ParametersConstants.CLOUD_ORIGIN, false)); if (maas.isEnable()) { //Deprecated. For backward compatibility. New name MAAS_EXTERNAL_ROUTE map.put("MAAS_SERVICE_ADDRESS", maas.getMaasUrl()); @@ -122,12 +124,12 @@ public Map getMap(String cloudName) { map.putIfAbsent("MAAS_CREDENTIALS_USERNAME", ((UsernamePasswordCredentials) cred).getUsername()); map.putIfAbsent("MAAS_CREDENTIALS_PASSWORD", ((UsernamePasswordCredentials) cred).getPassword()); } else { - map.putIfAbsent("MAAS_CREDENTIALS_USERNAME", DEFAULT_MAAS_LOGIN); - map.putIfAbsent("MAAS_CREDENTIALS_PASSWORD", DEFAULT_MAAS_PASSWORD); + map.putIfAbsent("MAAS_CREDENTIALS_USERNAME", DEFAULT_MAAS_LOGIN, ENVGENE_DEFAULT); + map.putIfAbsent("MAAS_CREDENTIALS_PASSWORD", DEFAULT_MAAS_PASSWORD,ENVGENE_DEFAULT); } } } else { - map.putIfAbsent("MAAS_ENABLED", new Parameter(false)); + map.putIfAbsent("MAAS_ENABLED", new Parameter(false, ParametersConstants.CLOUD_ORIGIN, false)); } Vault vaultConfig = config.getVault(); @@ -153,10 +155,10 @@ public Map getMap(String cloudName) { .getAuthClientToken(); map.put("VAULT_TOKEN", token); } catch (VaultException e) { - map.putIfAbsent("VAULT_TOKEN", ""); + map.putIfAbsent("VAULT_TOKEN", DEFAULT_EMPTY_STRING,ENVGENE_DEFAULT); } } else { - map.putIfAbsent("VAULT_TOKEN", ""); + map.putIfAbsent("VAULT_TOKEN", DEFAULT_EMPTY_STRING,ENVGENE_DEFAULT); } } Consul consul = config.getConsul(); @@ -168,11 +170,11 @@ public Map getMap(String cloudName) { if (cred instanceof StringCredentials) { map.putIfAbsent("CONSUL_ADMIN_TOKEN", ((StringCredentials) cred).getSecret()); } else { - map.putIfAbsent("CONSUL_ADMIN_TOKEN", ""); + map.putIfAbsent("CONSUL_ADMIN_TOKEN", DEFAULT_EMPTY_STRING,ENVGENE_DEFAULT); } } - map.put("PRODUCTION_MODE", new Parameter(config.isProductionMode())); + map.put("PRODUCTION_MODE", new Parameter(config.isProductionMode(), ParametersConstants.CLOUD_ORIGIN, false)); map.put("namespace", new Parameter(new NamespaceMap(tenant, cloudName, defaultNamespace, defaultApp, binding, originalNamespace).init())); map.put("CLOUDNAME", cloudName); map.put("e2e", new Parameter(e2e)); @@ -187,11 +189,12 @@ public Map getMap(String cloudName) { // Deprecated deployer parameters map.putIfAbsent("CUSTOM_HOST", customHost); map.putIfAbsent("SERVER_HOSTNAME", cloudHostname); - map.putIfAbsent("OPENSHIFT_SERVER", api); + map.putIfAbsent("OPENSHIFT_SERVER", api,ENVGENE_CALCULATED); // Deployer parameters String protocol = StringUtils.isNotBlank(config.getClProtocol()) ? config.getClProtocol() : "https"; - map.putIfAbsent("CLOUD_PROTOCOL", protocol.toLowerCase()); + String protocolOrigin = "https".equalsIgnoreCase(protocol) ? ENVGENE_CALCULATED : ParametersConstants.CLOUD_ORIGIN; + map.putIfAbsent("CLOUD_PROTOCOL", protocol.toLowerCase(),protocolOrigin); map.putIfAbsent("CLOUD_API_HOST", config.getCloudApiUrl()); if (StringUtils.isBlank(config.getCloudUrlPrv())) { map.putIfAbsent("CLOUD_PRIVATE_HOST", config.getCloudUrlPub()); @@ -201,7 +204,8 @@ public Map getMap(String cloudName) { map.putIfAbsent("CLOUD_PUBLIC_HOST", config.getCloudUrlPub()); String port = StringUtils.isNotBlank(config.getCloudApiPort()) ? config.getCloudApiPort() : "8443"; - map.putIfAbsent("CLOUD_API_PORT ", port); + String portOrigin = "8443".equalsIgnoreCase(port) ? ENVGENE_CALCULATED : ParametersConstants.CLOUD_ORIGIN; + map.putIfAbsent("CLOUD_API_PORT ", port, portOrigin); maps.put(cloudName, map); diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/EscapeMap.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/EscapeMap.java index 94b85c4d1..9f81da90d 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/EscapeMap.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/EscapeMap.java @@ -61,14 +61,43 @@ public Parameter put(String key, Map map) { return super.put(key, new Parameter(map, origin, false)); } + public Parameter put(String key, Map map,String origin) { + return super.put(key, new Parameter(map, origin, false)); + } + public Parameter put(String key, String string) { return super.put(key, new Parameter(string, origin, false)); } + public Parameter put(String key, String string,String origin) { + return super.put(key, new Parameter(string, origin, false)); + } + + + public Parameter put(String key, Object value) { + if (value == null) { + return super.put(key, new Parameter(null, origin, false)); + } + if (value instanceof Map map) { + return put(key, map); + } + if (value instanceof String string) { + return put(key, string); + } + if (value instanceof Parameter parameter) { + return super.put(key, parameter); + } + return super.put(key, new Parameter(value, origin, false)); + } + public Parameter putIfAbsent(String key, String string) { return super.putIfAbsent(key, new Parameter(string, origin, false)); } + public Parameter putIfAbsent(String key, String string, String origin) { + return super.putIfAbsent(key, new Parameter(string, origin, false)); + } + public void putAllStrings(Map m, String origin) { super.putAll(m.entrySet().stream() .map(entry -> new AbstractMap.SimpleEntry(entry.getKey(), new Parameter(entry.getValue(), origin, false))) diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/NamespaceApplicationMap.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/NamespaceApplicationMap.java index 1d3683fff..e6bdd1ca0 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/NamespaceApplicationMap.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/NamespaceApplicationMap.java @@ -26,12 +26,12 @@ import org.qubership.cloud.devops.commons.pojo.profile.model.Profile; import org.qubership.cloud.devops.commons.utils.BomReaderUtils; import org.qubership.cloud.devops.commons.utils.Parameter; -import org.qubership.cloud.devops.commons.utils.constant.ParametersConstants; import java.util.*; import static org.qubership.cloud.devops.commons.utils.constant.ApplicationConstants.*; +import static org.qubership.cloud.devops.commons.utils.constant.ParametersConstants.*; @Slf4j @@ -55,12 +55,8 @@ public Map getMap(String appName) { Map appParams = applicationParams != null ? applicationParams.getAppParams() : new HashMap<>(); Map configServerParams = applicationParams != null ? applicationParams.getConfigServerParams() : new HashMap<>(); - EscapeMap map = new EscapeMap(appParams, binding, - String.format(ParametersConstants.NS_APP_ORIGIN, namespace.getCloud().getTenant().getName(), namespace.getCloud().getName(), - namespace.getName(), appName)); - EscapeMap configServerMap = new EscapeMap(configServerParams, binding, - String.format(ParametersConstants.NS_APP_CONFIG_SERVER_ORIGIN, namespace.getCloud().getTenant().getName(), namespace.getCloud().getName(), - namespace.getName(), appName)); + EscapeMap map = new EscapeMap(appParams, binding, APP_ORIGIN ); + EscapeMap configServerMap = new EscapeMap(configServerParams, binding, APP_ORIGIN); map.put("APPLICATION_NAME", appName); @@ -84,7 +80,7 @@ private void populateAdditionalParams(String appName, String appFileRef, EscapeM map.put("ARTIFACT_DESCRIPTOR_GROUP_ID", applicationBomDto.getGroupId()); map.put("ARTIFACT_DESCRIPTOR_VERSION", applicationBomDto.getVersion()); map.put("ARTIFACT_DESCRIPTOR_MAVEN_REPO", applicationBomDto.getMavenRepo()); - map.put("DEPLOYMENT_SESSION_ID", applicationBomDto.getDeployerSessionId()); + map.put("DEPLOYMENT_SESSION_ID", applicationBomDto.getDeployerSessionId(),ENVGENE_PIPELINE_PARAMETER); map.put(APPR_CHART_NAME, applicationBomDto.getAppChartName()); map.put(SERVICES, applicationBomDto.getServices()); map.put(CONFIGURATIONS, applicationBomDto.getConfigurations()); diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/NamespaceMap.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/NamespaceMap.java index 74d9f1fe6..03af3016c 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/NamespaceMap.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/NamespaceMap.java @@ -75,7 +75,7 @@ public Map getMap(String namespaceName) { BgDomainEntityDTO bgDomainEntityDTO = inputDataService.getBGDomainData(); if (config != null) { mergeE2E = config.isMergeCustomPramsAndE2EParams(); - EscapeMap map = new EscapeMap(config.getCustomParameters(), binding, String.format(ParametersConstants.NS_ORIGIN, tenant, this.cloud, namespaceName)); + EscapeMap map = new EscapeMap(config.getCustomParameters(), binding, ParametersConstants.NS_ORIGIN); map.putIfAbsent(NAMESPACE, originalNamespace); map.put(APP, new Parameter(new NamespaceApplicationMap(config, defaultApp, binding).init())); @@ -90,8 +90,8 @@ public Map getMap(String namespaceName) { String customHost = StringUtils.isNotBlank(privateDNS) ? privateDNS : StringUtils.isNotBlank(publicDNS) ? publicDNS : cloudHostname; // Deprecated deployer parameters - map.putIfAbsent(GATEWAY_URL, "http://internal-gateway-service:8080"); - map.putIfAbsent(STATIC_CACHE_SERVICE_ROUTE_HOST, String.format("static-cache-service-%s.%s", originalNamespace, cloudHostname)); + map.putIfAbsent(GATEWAY_URL, "http://internal-gateway-service:8080", ParametersConstants.ENVGENE_DEFAULT); + map.putIfAbsent(STATIC_CACHE_SERVICE_ROUTE_HOST, String.format("static-cache-service-%s.%s", originalNamespace, cloudHostname),ParametersConstants.ENVGENE_CALCULATED); CredentialUtils credentialUtils = Injector.getInstance().getDi().get(CredentialUtils.class); @@ -104,12 +104,12 @@ public Map getMap(String namespaceName) { //Primary Namespace & Secondary Namespace if (origin.getName().equalsIgnoreCase(originalNamespace)) { - map.put(ORIGIN_NAMESPACE, originalNamespace); + map.put(ORIGIN_NAMESPACE, originalNamespace, ParametersConstants.ENVGENE_DEFAULT); map.put(PEER_NAMESPACE, peer.getName()); map.put(CONTROLLER_NAMESPACE, controller.getName()); } else if (controller.getName().equalsIgnoreCase(originalNamespace)) { //Controller Namespace if (origin != null) { - map.put(ORIGIN_NAMESPACE, origin.getName()); + map.put(ORIGIN_NAMESPACE, origin.getName()); } else { map.put(ORIGIN_NAMESPACE, originalNamespace); } @@ -121,7 +121,7 @@ public Map getMap(String namespaceName) { map.put(BG_CONTROLLER_URL, controller.getUrl()); } else { String bg_url = String.format("%s://bluegreen-controller-%s.%s", protocol.toLowerCase(), originalNamespace, customHost); - map.put(BG_CONTROLLER_URL, bg_url); + map.put(BG_CONTROLLER_URL, bg_url,ParametersConstants.ENVGENE_CALCULATED); } if (controller.getCredentials() != null && !controller.getCredentials().isEmpty()) { @@ -132,21 +132,21 @@ public Map getMap(String namespaceName) { map.put(BG_CONTROLLER_PASSWORD, usernamePasswordCredentials.getPassword()); } } else { - map.put(BG_CONTROLLER_LOGIN, "bgoperator"); - map.put(BG_CONTROLLER_PASSWORD, "F21wuZNRpw"); + map.put(BG_CONTROLLER_LOGIN, "bgoperator",ParametersConstants.ENVGENE_DEFAULT); + map.put(BG_CONTROLLER_PASSWORD, "F21wuZNRpw",ParametersConstants.ENVGENE_DEFAULT); } } // String rootUrl = Injector.getInstance().get(URLUtils.class).getRootUrl(); // rootUrl = rootUrl.endsWith("/") ? rootUrl.substring(0, rootUrl.length() - 1) : rootUrl; - map.put(CMDB_CALLBACK_TOKEN, ""); - map.put(CMDB_CALLBACK_URL, ""); + map.put(CMDB_CALLBACK_TOKEN, "",ParametersConstants.ENVGENE_DEFAULT); + map.put(CMDB_CALLBACK_URL, "",ParametersConstants.ENVGENE_DEFAULT); // map.put(CMDB_CALLBACK_TOKEN, Injector.getInstance().get(BGDomainRepository.class).getBgDomainToken(tenant, cloud, config.getBgDomainNamespaceData().getName())); // map.put(CMDB_CALLBACK_URL, String.format("%s/cm/v1/tenants/%s/clouds/%s/bgdomains/clone", // rootUrl, tenant, cloud)); } } else { - map.put(ORIGIN_NAMESPACE, originalNamespace); + map.put(ORIGIN_NAMESPACE, originalNamespace); } if (compositeStructureDTO != null) { @@ -159,8 +159,8 @@ public Map getMap(String namespaceName) { } // Deprecated deployer parameters - map.putIfAbsent(GATEWAY_URL, "http://internal-gateway-service:8080"); - map.putIfAbsent(STATIC_CACHE_SERVICE_ROUTE_HOST, String.format("static-cache-service-%s.%s", originalNamespace, cloudHostname)); + map.putIfAbsent(GATEWAY_URL, "http://internal-gateway-service:8080", ParametersConstants.ENVGENE_DEFAULT); + map.putIfAbsent(STATIC_CACHE_SERVICE_ROUTE_HOST, String.format("static-cache-service-%s.%s", originalNamespace, cloudHostname),ParametersConstants.ENVGENE_CALCULATED); String gatewayNamespace = ""; String idpUrlNamespace = ""; @@ -192,7 +192,7 @@ public Map getMap(String namespaceName) { addGatewayIdentityUrls(config.getCustomParameters(), map, false, protocol.toLowerCase(), customHost, gatewayNamespace, idpUrlNamespace); addGatewayIdentityUrls(config.getCustomParameters(), map, true, protocol.toLowerCase(), cloudHostname, gatewayNamespace, idpUrlNamespace); // map.putIfAbsent(SSL_SECRET, "defaultsslcertificate"); setting this value after being validated finally - map.putIfAbsent(BUILD_TAG_NEW, "keycloak-database"); + map.putIfAbsent(BUILD_TAG_NEW, "keycloak-database", ParametersConstants.ENVGENE_DEFAULT); if (binding.getDeployerInputs() != null) { map.put("CLIENT_PREFIX", originalNamespace); if (binding.getDeployerInputs().getSecretId() != null) { @@ -201,8 +201,8 @@ public Map getMap(String namespaceName) { } } - EscapeMap e2e = new EscapeMap(config.getE2eParameters(), binding, String.format(ParametersConstants.NS_E2E_ORIGIN, tenant, this.cloud, namespaceName)); - EscapeMap configServer = new EscapeMap(config.getConfigServerParameters(), binding, String.format(ParametersConstants.NS_CONFIG_SERVER_ORIGIN, tenant, this.cloud, namespaceName)); + EscapeMap e2e = new EscapeMap(config.getE2eParameters(), binding,ParametersConstants.NS_ORIGIN); + EscapeMap configServer = new EscapeMap(config.getConfigServerParameters(), binding, ParametersConstants.NS_ORIGIN); map.put(E2E, new Parameter(e2e)); map.put(CONFIG_SERVER, new Parameter(configServer)); @@ -215,14 +215,14 @@ public Map getMap(String namespaceName) { private void setBaselineVars(EscapeMap map, CompositeEntityDTO baselineEntity, BgDomainEntityDTO bgDomainEntityDTO) { if (baselineEntity != null) { if (baselineEntity.getType().equalsIgnoreCase(BG_DOMAIN) && bgDomainEntityDTO != null) { - map.put(BASELINE_ORIGIN, bgDomainEntityDTO.getOriginNamespace().getName()); - map.put(BASELINE_PEER, bgDomainEntityDTO.getPeerNamespace().getName()); - map.put(BASELINE_CONTROLLER, bgDomainEntityDTO.getControllerNamespace().getName()); - map.put(BASELINE_PROJ, bgDomainEntityDTO.getControllerNamespace().getName()); + map.put(BASELINE_ORIGIN, bgDomainEntityDTO.getOriginNamespace().getName(),ParametersConstants.BG_DOMAIN); + map.put(BASELINE_PEER, bgDomainEntityDTO.getPeerNamespace().getName(),ParametersConstants.BG_DOMAIN); + map.put(BASELINE_CONTROLLER, bgDomainEntityDTO.getControllerNamespace().getName(),ParametersConstants.BG_DOMAIN); + map.put(BASELINE_PROJ, bgDomainEntityDTO.getControllerNamespace().getName(),ParametersConstants.BG_DOMAIN); } else if (baselineEntity.getType().equalsIgnoreCase(NAMESPACE) && !baselineEntity.getName().equalsIgnoreCase(originalNamespace)) { - map.put(BASELINE_ORIGIN, baselineEntity.getName()); - map.put(BASELINE_PROJ, baselineEntity.getName()); + map.put(BASELINE_ORIGIN, baselineEntity.getName(),ParametersConstants.COMPOSITE_STRUCTURE); + map.put(BASELINE_PROJ, baselineEntity.getName(),ParametersConstants.COMPOSITE_STRUCTURE); } } } @@ -238,8 +238,8 @@ private void setSecretValues(EscapeMap map, CredentialUtils credentialUtils) { } credentialValue = ca.getSecret(); credentialValue = credentialValue.replace("\r\n", "\n"); - map.put("SSL_SECRET_VALUE", credentialValue); - map.put("CERTIFICATE_BUNDLE_MD5SUM", DigestUtils.md5Hex(DigestUtils.getMd5Digest().digest(credentialValue.getBytes(StandardCharsets.UTF_8)))); + map.put("SSL_SECRET_VALUE", credentialValue,ParametersConstants.ENVGENE_CALCULATED); + map.put("CERTIFICATE_BUNDLE_MD5SUM", DigestUtils.md5Hex(DigestUtils.getMd5Digest().digest(credentialValue.getBytes(StandardCharsets.UTF_8))),ParametersConstants.ENVGENE_CALCULATED); } } @@ -252,19 +252,19 @@ private void addGatewayIdentityUrls(Map customParameters, Escape String gatewayUrl = isPublic ? PUBLIC_GATEWAY_URL : PRIVATE_GATEWAY_URL; String identityProviderUrl = isPublic ? PUBLIC_IDENTITY_PROVIDER_URL : PRIVATE_IDENTITY_PROVIDER_URL; if (customParameters.containsKey(gatewayUrl) && customParameters.containsKey(identityProviderUrl)) { - map.put(gatewayUrl, String.valueOf(customParameters.get(gatewayUrl))); - map.put(identityProviderUrl, String.valueOf(customParameters.get(identityProviderUrl))); + map.put(gatewayUrl, String.valueOf(customParameters.get(gatewayUrl)),ParametersConstants.CUSTOM_PARAMS_ORIGIN); + map.put(identityProviderUrl, String.valueOf(customParameters.get(identityProviderUrl)),ParametersConstants.CUSTOM_PARAMS_ORIGIN); } else { if (customParameters.containsKey(gatewayUrl)) { - map.put(gatewayUrl, String.valueOf(customParameters.get(gatewayUrl))); - map.put(identityProviderUrl, String.valueOf(customParameters.get(gatewayUrl))); + map.put(gatewayUrl, String.valueOf(customParameters.get(gatewayUrl)),ParametersConstants.CUSTOM_PARAMS_ORIGIN); + map.put(identityProviderUrl, String.valueOf(customParameters.get(gatewayUrl)),ParametersConstants.CUSTOM_PARAMS_ORIGIN); } if (customParameters.containsKey(identityProviderUrl)) { - map.put(identityProviderUrl, String.valueOf(customParameters.get(identityProviderUrl))); + map.put(identityProviderUrl, String.valueOf(customParameters.get(identityProviderUrl)),ParametersConstants.CUSTOM_PARAMS_ORIGIN); } } - map.putIfAbsent(gatewayUrl, defaultGatewayUrl); - map.putIfAbsent(identityProviderUrl, defaultIdpUrl); + map.putIfAbsent(gatewayUrl, defaultGatewayUrl,ParametersConstants.ENVGENE_CALCULATED); + map.putIfAbsent(identityProviderUrl, defaultIdpUrl,ParametersConstants.ENVGENE_CALCULATED); } } diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/TenantMap.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/TenantMap.java index acde3c677..1c0c50240 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/TenantMap.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/TenantMap.java @@ -59,10 +59,9 @@ public Map getMap(String tenantName) { mergeE2E = config.getGlobalParameters().getE2eParameters().isMergeTenantAndE2EParams(); EscapeMap map = new EscapeMap(config .getGlobalParameters() - .getDeployParameters(), binding, - String.format(ParametersConstants.TENANT_ORIGIN, tenantName)); - EscapeMap e2e = new EscapeMap(config.getGlobalParameters().getE2eParameters().getEnvParameters(), binding, String.format(ParametersConstants.TENANT_E2E_ORIGIN, tenantName)); - EscapeMap configServer = new EscapeMap(config.getGlobalParameters().getTechnicalConfiguration(), binding, String.format(ParametersConstants.TENANT_CONFIG_SERVER_ORIGIN, tenantName)); + .getDeployParameters(), binding, ParametersConstants.TENANT_ORIGIN); + EscapeMap e2e = new EscapeMap(config.getGlobalParameters().getE2eParameters().getEnvParameters(), binding, ParametersConstants.TENANT_ORIGIN); + EscapeMap configServer = new EscapeMap(config.getGlobalParameters().getTechnicalConfiguration(), binding,ParametersConstants.TENANT_ORIGIN); map.put("cloud", new Parameter(new CloudMap(tenantName, defaultCloud, defaultNamespace, defaultApp, binding, originalNamespace).init())); map.put("e2e", new Parameter(e2e)); diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/service/ParametersCalculationServiceV2.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/service/ParametersCalculationServiceV2.java index 34fd04841..d05070697 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/service/ParametersCalculationServiceV2.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/service/ParametersCalculationServiceV2.java @@ -24,6 +24,7 @@ import org.qubership.cloud.devops.commons.pojo.parameterset.CustomParameterDTO; import org.qubership.cloud.devops.commons.utils.Parameter; import org.qubership.cloud.devops.commons.utils.ParameterUtils; +import org.qubership.cloud.devops.commons.utils.constant.ParametersConstants; import org.qubership.cloud.parameters.processor.ParametersProcessor; import org.qubership.cloud.parameters.processor.dto.DeployerInputs; import org.qubership.cloud.parameters.processor.dto.ParameterBundle; @@ -52,7 +53,7 @@ public ParametersCalculationServiceV2(ParametersProcessor parametersProcessor) { public ParameterBundle getCliParameter(String tenantName, String cloudName, String namespaceName, String applicationName, DeployerInputs deployerInputs, String originalNamespace, - Map k8TokenMap, CustomParameterDTO customParams) { + Map k8TokenMap, CustomParameterDTO customParams) { return getParameterBundle(tenantName, cloudName, namespaceName, applicationName, deployerInputs, originalNamespace, k8TokenMap, customParams); @@ -64,7 +65,7 @@ public ParameterBundle getCliE2EParameter(String tenantName, String cloudName) { public ParameterBundle getCleanupParameterBundle(String tenantName, String cloudName, String namespaceName, DeployerInputs deployerInputs, String originalNamespace, - Map k8TokenMap) { + Map k8TokenMap) { Params parameters = parametersProcessor.processNamespaceParameters(tenantName, cloudName, namespaceName, @@ -79,7 +80,7 @@ public ParameterBundle getCleanupParameterBundle(String tenantName, String cloud private ParameterBundle getParameterBundle(String tenantName, String cloudName, String namespaceName, String applicationName, DeployerInputs deployerInputs, String originalNamespace, - Map k8TokenMap, CustomParameterDTO customParams) { + Map k8TokenMap, CustomParameterDTO customParams) { Params parameters = parametersProcessor.processAllParameters(tenantName, cloudName, namespaceName, @@ -98,8 +99,10 @@ private ParameterBundle getParameterBundle(String tenantName, String cloudName, } if (MapUtils.isNotEmpty(customParams.getAllParams())) { prepareCustomParams(customParams, parameters.getDeployParams(), parameters.getTechParams()); - parameterBundle.setCustomDeployParameters(ParametersProcessor.convertParameterMapToObject(customParams.getDeployParams())); - parameterBundle.setCustomTechParameters(ParametersProcessor.convertParameterMapToObject(customParams.getTechnicalParams())); + parameterBundle.setCustomDeployParameters( + ParameterUtils.deepSortMapKeysPreservingParameters(customParams.getDeployParams())); + parameterBundle.setCustomTechParameters( + ParameterUtils.deepSortMapKeysPreservingParameters(customParams.getTechnicalParams())); } prepareSecureInsecureParams(parameters.getDeployParams(), parameterBundle, ParameterType.DEPLOY, k8TokenMap, originalNamespace); prepareSecureInsecureParams(parameters.getTechParams(), parameterBundle, ParameterType.TECHNICAL, k8TokenMap, originalNamespace); @@ -108,18 +111,25 @@ private ParameterBundle getParameterBundle(String tenantName, String cloudName, public Map getProcessedParameters(Map parameters) { Map processedParameters = parametersProcessor.processParameters(parameters); - return ParametersProcessor.convertParameterMapToObject(processedParameters); + return ParameterUtils.deepSortMapKeysPreservingParameters(processedParameters); } private static void processPerServiceParams(Params parameters, ParameterBundle parameterBundle) { Parameter parameter = parameters.getDeployParams().get(PER_SERVICE_DEPLOY_PARAMS); - if (parameter.getValue() == null) { - parameters.getDeployParams().remove(PER_SERVICE_DEPLOY_PARAMS); + if (parameter == null || parameter.getValue() == null) { + if (parameter != null) { + parameters.getDeployParams().remove(PER_SERVICE_DEPLOY_PARAMS); + } + parameterBundle.setPerServiceParams(new HashMap<>()); + return; + } + Object paramValue = parameter.getValue(); + if (!(paramValue instanceof Map)) { parameterBundle.setPerServiceParams(new HashMap<>()); return; } parameterBundle.setProcessPerServiceParams(true); - Map perServiceParams = ParametersProcessor.convertParameterMapToObject((Map) parameter.getValue()); + Map perServiceParams = ParameterUtils.deepSortMapKeysPreservingParameters((Map) paramValue); parameterBundle.setPerServiceParams(perServiceParams); parameters.getDeployParams().remove(PER_SERVICE_DEPLOY_PARAMS); @@ -127,24 +137,49 @@ private static void processPerServiceParams(Params parameters, ParameterBundle p private static void processDeploymentDescriptorParams(Params parameters, ParameterBundle parameterBundle) { Parameter commParameter = parameters.getDeployParams().get(COMMON_DEPLOY_DESC); - if (commParameter.getValue() == null) { + if (commParameter == null || commParameter.getValue() == null) { + if (commParameter != null) { + parameters.getDeployParams().remove(COMMON_DEPLOY_DESC); + } parameters.getDeployParams().remove(COMMON_DEPLOY_DESC); parameters.getDeployParams().remove(DEPLOY_DESC); parameterBundle.setDeployDescParams(new HashMap<>()); return; } Parameter parameter = parameters.getDeployParams().get(DEPLOY_DESC); - if (parameter.getValue() == null) { + Object paramValue = parameter != null ? parameter.getValue() : null; + if (paramValue == null) { parameters.getDeployParams().remove(DEPLOY_DESC); } Map finalDeployDescMap = new LinkedHashMap<>(); - Map deployDescParams = ParametersProcessor.convertParameterMapToObject((Map) parameter.getValue()); - - - Map commonParamMap = new LinkedHashMap<>(); - Map commonDepDescMap = ParametersProcessor.convertParameterMapToObject((Map) commParameter.getValue()); - commonDepDescMap.entrySet().stream().forEach(entry -> commonParamMap.putAll((Map) entry.getValue())); - + Map deployDescParams = (paramValue instanceof Map) ? + ParameterUtils.deepSortMapKeysPreservingParameters((Map) paramValue) : new LinkedHashMap<>(); + + Map commonParamMapAccumulated = new LinkedHashMap<>(); + Object commParamValue = commParameter.getValue(); + Map commonDepDescMap = (commParamValue instanceof Map) ? + ParameterUtils.deepSortMapKeysPreservingParameters((Map) commParamValue) : new LinkedHashMap<>(); + commonDepDescMap.entrySet().stream().forEach(entry -> { + Object value = entry.getValue(); + Map valueMap = null; + if (value instanceof Parameter) { + Object innerValue = ((Parameter) value).getValue(); + if (innerValue instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) innerValue; + valueMap = map; + } + } else if (value instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) value; + valueMap = map; + } + if (valueMap != null) { + commonParamMapAccumulated.putAll(valueMap); + } + }); + Map commonParamMap = + ParameterUtils.deepSortMapKeysPreservingParameters(commonParamMapAccumulated); Map deployDescParamMap = new LinkedHashMap<>(); deployDescParamMap.put("deployDescriptor", deployDescParams); @@ -175,7 +210,7 @@ private ParameterBundle getE2EParameterBundle(String tenantName, String cloudNam } public void prepareSecureInsecureParams(Map parameters, ParameterBundle parameterBundle - , ParameterType parameterType, Map k8TokenMap, String originalNamespace) { + , ParameterType parameterType, Map k8TokenMap, String originalNamespace) { Map securedParams = new TreeMap<>(); Map inSecuredParams = new TreeMap<>(); if (MapUtils.isEmpty(parameters) && MapUtils.isEmpty(parameterBundle.getCustomTechParameters())) { @@ -184,8 +219,8 @@ public void prepareSecureInsecureParams(Map parameters, Param } filterSecuredParams(parameters, securedParams, inSecuredParams, parameterType); - Map finalSecuredParams = ParametersProcessor.convertParameterMapToObject(securedParams); - Map inSecuredParamsAsObject = ParametersProcessor.convertParameterMapToObject(inSecuredParams); + Map finalSecuredParams = ParameterUtils.deepSortMapKeysPreservingParameters(securedParams); + Map inSecuredParamsAsObject = ParameterUtils.deepSortMapKeysPreservingParameters(inSecuredParams); if (parameterType == ParameterType.E2E) { parameterBundle.setSecuredE2eParams(finalSecuredParams); parameterBundle.setE2eParams(inSecuredParamsAsObject); @@ -205,24 +240,31 @@ private static void prepareCustomTechSecureParams(ParameterBundle parameterBundl if (MapUtils.isEmpty(parameterBundle.getCustomTechParameters())) { return; } - Map customTechParams = ParametersProcessor.convertParameterMapToObject(parameterBundle.getCustomTechParameters()); + Map customTechParams = + ParameterUtils.deepSortMapKeysPreservingParameters(parameterBundle.getCustomTechParameters()); if (MapUtils.isEmpty(finalSecuredParams)) { - parameterBundle.setSecuredConfigParams(new TreeMap<>(customTechParams)); + parameterBundle.setSecuredConfigParams(customTechParams); } else { parameterBundle.getSecuredConfigParams().putAll(customTechParams); + parameterBundle.setSecuredConfigParams( + ParameterUtils.deepSortMapKeysPreservingParameters(parameterBundle.getSecuredConfigParams())); } } - private void handleDeployParameters(ParameterBundle parameterBundle, Map k8TokenMap, String originalNamespace, Map finalSecuredParams, Map inSecuredParamsAsObject) { - Object appChartName = inSecuredParamsAsObject.get(APPR_CHART_NAME); - parameterBundle.setAppChartName(appChartName != null ? appChartName.toString() : ""); - inSecuredParamsAsObject.remove(APPR_CHART_NAME); //remove app chart name from parameters once after the usage + private void handleDeployParameters(ParameterBundle parameterBundle, Map k8TokenMap, String originalNamespace, + Map finalSecuredParams, Map inSecuredParamsAsObject) { + Object appChartNameObj = inSecuredParamsAsObject.get(APPR_CHART_NAME); + if (appChartNameObj instanceof Parameter) { + appChartNameObj = ((Parameter) appChartNameObj).getValue(); + } + parameterBundle.setAppChartName(appChartNameObj != null ? appChartNameObj.toString() : ""); + inSecuredParamsAsObject.remove(APPR_CHART_NAME); Map deployCollisionParams = getCollisionParams(inSecuredParamsAsObject); Map securedCollisionParams = getCollisionParams(finalSecuredParams); parameterBundle.setCollisionDeployParameters(deployCollisionParams); parameterBundle.setCollisionSecureParameters(securedCollisionParams); - copyParams(finalSecuredParams, inSecuredParamsAsObject, k8TokenMap, originalNamespace); - prepareBundleParameters(finalSecuredParams, inSecuredParamsAsObject); + copyParams(parameterBundle, finalSecuredParams, inSecuredParamsAsObject, k8TokenMap, originalNamespace); + prepareBundleParameters(parameterBundle, finalSecuredParams, inSecuredParamsAsObject); Map finalInsecureParams = prepareFinalParams(inSecuredParamsAsObject, parameterBundle.isProcessPerServiceParams(), deployCollisionParams); Map finalSecParams = prepareFinalParams(finalSecuredParams, true, securedCollisionParams); @@ -230,7 +272,7 @@ private void handleDeployParameters(ParameterBundle parameterBundle, Map finalSecParams, Map finalInsecureParams) { + private void prepareBundleParameters(ParameterBundle parameterBundle, Map finalSecParams, Map finalInsecureParams) { if (finalInsecureParams.containsKey(DEFAULT_SSL_CERTIFICATES_BUNDLE)) { Object defaultSslCertificatesBundle = finalInsecureParams.get(DEFAULT_SSL_CERTIFICATES_BUNDLE); finalSecParams.put(SSL_SECRET_VALUE, defaultSslCertificatesBundle); @@ -241,19 +283,22 @@ private void prepareBundleParameters(Map finalSecParams, Map finalSecParams, Map finalInsecureParams, - Map k8TokenMap, String originalNamespace) { + private void copyParams(ParameterBundle parameterBundle, Map finalSecParams, Map finalInsecureParams, + Map k8TokenMap, String originalNamespace) { SECURED_KEYS.stream() .filter(finalInsecureParams::containsKey) .forEach(key -> { finalSecParams.put(key, finalInsecureParams.get(key)); finalInsecureParams.remove(key); }); - finalSecParams.put(K8S_TOKEN, k8TokenMap.get(originalNamespace)); + Object k8Token = k8TokenMap.get(originalNamespace); + if (k8Token != null) { + finalSecParams.put(K8S_TOKEN, k8Token); + } } private Map getCollisionParams(Map parameters) { @@ -261,18 +306,26 @@ private Map getCollisionParams(Map parameters) { Map collisionParams = new LinkedHashMap<>(); if (parameters.containsKey(SERVICES)) { - serviceMap = (Map) parameters.get(SERVICES); + Object servicesObj = parameters.get(SERVICES); + if (servicesObj instanceof Parameter) { + servicesObj = ((Parameter) servicesObj).getValue(); + } + if (servicesObj instanceof Map) { + @SuppressWarnings("unchecked") + Map servicesMap = (Map) servicesObj; + serviceMap = servicesMap; + } } Set services = serviceMap.keySet(); Set keysToRemove = new HashSet<>(); parameters.forEach((key, value) -> { if (services.contains(key) && !entities.contains(key)) { collisionParams.put(key, value); - keysToRemove.add(key); // mark for removal + keysToRemove.add(key); } }); keysToRemove.forEach(parameters::remove); - return collisionParams; + return ParameterUtils.deepSortMapKeysPreservingParameters(collisionParams); } private Map prepareFinalParams(Map parameters, @@ -282,30 +335,66 @@ private Map prepareFinalParams(Map parameters, Map orderedMap = new LinkedHashMap<>(); entities.stream() - .map(key -> (Map) parameters.remove(key)) + .map(parameters::remove) .filter(Objects::nonNull) - .forEach(finalMap::putAll); - Map sortedMap = new TreeMap<>(parameters); + .forEach(value -> { + Map mapValue = null; + if (value instanceof Parameter) { + Object paramValue = ((Parameter) value).getValue(); + if (paramValue instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) paramValue; + mapValue = map; + } + } else if (value instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) value; + mapValue = map; + } + if (mapValue != null) { + finalMap.putAll(mapValue); + } + }); + Map sortedMap = ParameterUtils.deepSortMapKeysPreservingParameters(parameters); orderedMap.putAll(sortedMap); + + final Map sortedForLambdas; + if (parameters != null && !parameters.isEmpty() && !collisionParams.isEmpty()) { + Map merged = new LinkedHashMap<>(sortedMap); + merged.putAll(collisionParams); + sortedForLambdas = ParameterUtils.deepSortMapKeysPreservingParameters(merged); + } else { + sortedForLambdas = sortedMap; + } if (parameters != null && !parameters.isEmpty()) { - if (!collisionParams.isEmpty()) { - sortedMap.putAll(collisionParams); - } - orderedMap.put("global", sortedMap); + orderedMap.put("global", sortedForLambdas); } if (processPerServiceParams) { finalMap.forEach((key, value) -> { - if (value instanceof Map) { - finalMap.put(key, sortedMap); + Object mapValue = value instanceof Parameter ? ((Parameter) value).getValue() : value; + if (mapValue instanceof Map) { + finalMap.put(key, sortedForLambdas); } }); } else { finalMap.forEach((key, value) -> { - if (value instanceof Map) { - Map valueMap = (Map) value; - valueMap.put("!merge", sortedMap); - Map sortedValueMap = new TreeMap<>(valueMap); - finalMap.put(key, sortedValueMap); + Map valueMap = null; + if (value instanceof Parameter) { + Object paramValue = ((Parameter) value).getValue(); + if (paramValue instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) paramValue; + valueMap = map; + } + } else if (value instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) value; + valueMap = map; + } + if (valueMap != null) { + Map modifiedMap = ParameterUtils.deepSortMapKeysPreservingParameters(valueMap); + modifiedMap.put("!merge", sortedForLambdas); + finalMap.put(key, ParameterUtils.deepSortMapKeysPreservingParameters(modifiedMap)); } }); } diff --git a/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/expression/LanguageTest.java b/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/expression/LanguageTest.java index 76c33429c..93069b337 100644 --- a/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/expression/LanguageTest.java +++ b/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/expression/LanguageTest.java @@ -17,6 +17,7 @@ package org.qubership.cloud.expression; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.qubership.cloud.devops.commons.utils.constant.ParametersConstants.*; import org.qubership.cloud.BindingBaseTest; import org.qubership.cloud.devops.commons.utils.constant.ParametersConstants; @@ -153,32 +154,32 @@ public void assertMap(Map map) { assertEquals(checkMap, map); - String tenantOrigin = String.format(ParametersConstants.TENANT_ORIGIN, "tenant"); + String tenantOrigin = ParametersConstants.TENANT_ORIGIN; assertEquals(tenantOrigin, map.get("TENANT_PARAM").getOrigin()); - String appOrigin = String.format(ParametersConstants.APP_ORIGIN, "application"); + String appOrigin = APP_ORIGIN; assertEquals(appOrigin, map.get("APP_PARAM").getOrigin()); assertEquals(appOrigin, map.get("TENANT_PARAM_OVERRIDE_BY_APP").getOrigin()); - String cloudOrigin = String.format(ParametersConstants.CLOUD_ORIGIN, "tenant", "cloud"); + String cloudOrigin = ParametersConstants.CLOUD_ORIGIN; assertEquals(cloudOrigin, map.get("CLOUD_PARAM").getOrigin()); assertEquals(cloudOrigin, map.get("APP_PARAM_OVERRIDE_BY_CLOUD").getOrigin()); assertEquals(cloudOrigin, map.get("TENANT_PARAM_OVERRIDE_BY_CLOUD").getOrigin()); - String cloudAppOrigin = String.format(ParametersConstants.CLOUD_APP_ORIGIN, "tenant", "cloud", "application"); + String cloudAppOrigin = APP_ORIGIN; assertEquals(cloudAppOrigin, map.get("CLOUD_APP_PARAM").getOrigin()); assertEquals(cloudAppOrigin, map.get("TENANT_PARAM_OVERRIDE_BY_CLOUD_APP").getOrigin()); assertEquals(cloudAppOrigin, map.get("APP_PARAM_OVERRIDE_BY_CLOUD_APP").getOrigin()); assertEquals(cloudAppOrigin, map.get("CLOUD_PARAM_OVERRIDE_BY_CLOUD_APP").getOrigin()); - String nsOrigin = String.format(ParametersConstants.NS_ORIGIN, "tenant", "cloud", "namespace"); + String nsOrigin = ParametersConstants.NS_ORIGIN; assertEquals(nsOrigin, map.get("NS_PARAM").getOrigin()); assertEquals(nsOrigin, map.get("CLOUD_APP_PARAM_OVERRIDE_BY_NS").getOrigin()); assertEquals(nsOrigin, map.get("CLOUD_PARAM_OVERRIDE_BY_NS").getOrigin()); assertEquals(nsOrigin, map.get("APP_PARAM_OVERRIDE_BY_NS").getOrigin()); assertEquals(nsOrigin, map.get("TENANT_PARAM_OVERRIDE_BY_NS").getOrigin()); - String nsAppOrigin = String.format(ParametersConstants.NS_APP_ORIGIN, "tenant", "cloud", "namespace", "application"); + String nsAppOrigin = APP_ORIGIN; assertEquals(nsAppOrigin, map.get("NS_APP_PARAM").getOrigin()); assertEquals(nsAppOrigin, map.get("NS_PARAM_OVERRIDE_BY_NS_APP").getOrigin()); assertEquals(nsAppOrigin, map.get("CLOUD_APP_PARAM_OVERRIDE_BY_NS_APP").getOrigin()); diff --git a/build_pipegene/scripts/effective_set_job.py b/build_pipegene/scripts/effective_set_job.py index 0516a5d57..842016fcd 100644 --- a/build_pipegene/scripts/effective_set_job.py +++ b/build_pipegene/scripts/effective_set_job.py @@ -82,6 +82,8 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste cmdb_cli_cmd_call.extend(["$extra_args"]) if deployment_id: cmdb_cli_cmd_call.extend([f"--extra_params=DEPLOYMENT_SESSION_ID={deployment_id}"]) + if params.get("ES_TRACE_ENABLED").lower() == "true": + cmdb_cli_cmd_call.extend(["--enable-traceability=true"]) if custom_params: logger.info(f"custom_params : {custom_params}") diff --git a/scripts/utils/pipeline_parameters.py b/scripts/utils/pipeline_parameters.py index 9b90c006d..e9c3061eb 100644 --- a/scripts/utils/pipeline_parameters.py +++ b/scripts/utils/pipeline_parameters.py @@ -42,6 +42,7 @@ def get_pipeline_parameters() -> dict: "EFFECTIVE_SET_CONFIG" : getenv("EFFECTIVE_SET_CONFIG"), "ENV_INVENTORY_CONTENT": getenv("ENV_INVENTORY_CONTENT"), "CUSTOM_PARAMS" : getenv("CUSTOM_PARAMS"), + "ES_TRACE_ENABLED": getenv("ES_TRACE_ENABLED","false"), "ENV_TEMPLATE_VERSION_UPDATE_MODE": getenv( "ENV_TEMPLATE_VERSION_UPDATE_MODE") or TemplateVersionUpdateMode.PERSISTENT.value, }