diff --git a/python/envgene/envgenehelper/creds_helper.py b/python/envgene/envgenehelper/creds_helper.py index 2334a5505..9bf545235 100644 --- a/python/envgene/envgenehelper/creds_helper.py +++ b/python/envgene/envgenehelper/creds_helper.py @@ -1,8 +1,10 @@ import re +import os from pathlib import Path from envgenehelper import crypt, getenv_with_error, get_env_instances_dir, findAllYamlsInDir, openYaml, getEnvCredentialsPath from envgenehelper.errors import ValidationError +from envgenehelper.yaml_helper import store_value_to_yaml, writeYamlToFile, beautifyYaml, yaml from .logger import logger @@ -251,7 +253,7 @@ def validate_creds(creds_path: str = ""): def check_cred_value(credId, credValue) -> str: result = "" type = credValue["type"] - data = credValue["data"] + data = credValue.get("data", {}) match type: case _ if type == CRED_TYPE_USERPASS: if is_envgenenullvalue(data["username"]) or is_envgenenullvalue(data["password"]): @@ -273,3 +275,44 @@ def is_envgenenullvalue(value: object) -> bool: if value.lower() == "envgenenullvalue": return True return False + +def extract_external_cred(cred_map): + if cred_map.get("$type") != "credRef": + return None + cred_id = cred_map.get("credId") + if not cred_id or not str(cred_id).strip(): + raise ValueError(f"Invalid credRef: 'credId' is missing or empty in {cred_map}") + return cred_id + +def validate_cred_types(credsMap, isExternalCredEnv, credFile): + types = { + v.get("type") + for v in credsMap.values() + if isinstance(v, dict) and v.get("type") + } + if not types: + return + if isExternalCredEnv: + if types != {"external"}: + raise ValueError(f"Only external credentials allowed. Found: {types} in {credFile}") + else: + if "external" in types: + raise ValueError(f"External credentials not allowed. Found: {types} in {credFile}") + +def has_external_creds(credsMap): + return any( + isinstance(v, dict) and v.get("type") == "external" + for v in credsMap.values() + ) + +def copy_creds_to_env_creds_file(env_dir, credsYamlContent, comment, credsSchema): + envCredentialsPath = f"{env_dir}/Credentials/credentials.yml" + if os.path.exists(envCredentialsPath) : + envCredsYaml = openYaml(envCredentialsPath) + else: + envCredsYaml = yaml.load("{}") + for key, value in credsYamlContent.items() : + store_value_to_yaml(envCredsYaml, key, value, comment) + # storing credentials yaml + writeYamlToFile(envCredentialsPath, envCredsYaml) + beautifyYaml(envCredentialsPath, credsSchema) \ No newline at end of file diff --git a/python/envgene/envgenehelper/yaml_helper.py b/python/envgene/envgenehelper/yaml_helper.py index 345abb5f5..041e58b42 100644 --- a/python/envgene/envgenehelper/yaml_helper.py +++ b/python/envgene/envgenehelper/yaml_helper.py @@ -141,10 +141,7 @@ def alignYamlComments(yamlContent, extra_indent=0): def sortYaml(yaml_data, schema_path, remove_additional_props): - with open(schema_path, 'r') as f: - schema_data = json.load(f) - logger.debug(f'Checking yaml with schema: {schema_path}') - jsonschema.validate(yaml_data, schema_data) + schema_data = validateSchema(yaml_data, schema_path) sort_data = jschon_tools.process_json_doc( schema_data=schema_data, doc_data=yaml_data, @@ -153,6 +150,13 @@ def sortYaml(yaml_data, schema_path, remove_additional_props): ) return sort_data +def validateSchema(yaml_data, schema_path): + with open(schema_path, 'r') as f: + schema_data = json.load(f) + logger.debug(f'Checking yaml with schema: {schema_path}') + jsonschema.validate(yaml_data, schema_data) + return schema_data + def get_nested_yaml_attribute_or_fail(yaml_content, attribute_str): keys = attribute_str.split('.') diff --git a/schemas/application.sbom.schema.json b/schemas/application.sbom.schema.json index b846bca99..40c700046 100644 --- a/schemas/application.sbom.schema.json +++ b/schemas/application.sbom.schema.json @@ -992,4 +992,4 @@ } } } -} +} \ No newline at end of file diff --git a/schemas/credential.schema.json b/schemas/credential.schema.json index 10c74c988..a369e2ddc 100644 --- a/schemas/credential.schema.json +++ b/schemas/credential.schema.json @@ -168,4 +168,4 @@ } ] } -} +} \ No newline at end of file diff --git a/scripts/build_env/build_env.py b/scripts/build_env/build_env.py index dd40aec66..b76c4fe5a 100644 --- a/scripts/build_env/build_env.py +++ b/scripts/build_env/build_env.py @@ -466,7 +466,7 @@ def process_additional_template_parameters(render_env_dir, source_env_dir, all_i def build_env(env_name, env_instances_dir, parameters_dir, env_template_dir, resource_profiles_dir, - env_specific_resource_profile_map, all_instances_dir, render_context, templates_dirs=None): + env_specific_resource_profile_map, all_instances_dir, render_context, templates_dirs=None, isExternalCredEnv=False): # Check which role-specific templates were downloaded templates_dirs = templates_dirs or {} origin_template_exists = NamespaceRole.ORIGIN in templates_dirs @@ -525,7 +525,7 @@ def build_env(env_name, env_instances_dir, parameters_dir, env_template_dir, res header_text=generated_header_text, process_env_specific=False) # process cloud passport - process_cloud_passport(env_dir, env_instances_dir, all_instances_dir) + process_cloud_passport(env_dir, env_instances_dir, all_instances_dir, isExternalCredEnv) logger.info("Processing cloud with env specific parameters.") processTemplate( cloudTemlatePath, diff --git a/scripts/build_env/cloud_passport.py b/scripts/build_env/cloud_passport.py index f4c11f8fa..9e43a7e70 100644 --- a/scripts/build_env/cloud_passport.py +++ b/scripts/build_env/cloud_passport.py @@ -20,13 +20,15 @@ def process_and_update_key(targetKey, targetYaml, sourceKey, sourceYaml, comment merge_dict_key_with_comment(targetKey, targetYaml, sourceKey, sourceYaml, comment) del sourceYaml[sourceKey] -def mergeDeployParametersFromPassport(cloudPassportYaml, cloudYaml, comment) : +def mergeDeployParametersFromPassport(cloudPassportYaml, cloudYaml, comment, isExternalCredEnv) : for domain in cloudPassportYaml: if domain == "version": continue for paramKey, paramValue in cloudPassportYaml[domain].items(): + if isExternalCredEnv and paramKey == "SECRET_FLOW" and paramKey in cloudYaml["deployParameters"]: + continue store_value_to_yaml(cloudYaml["deployParameters"], paramKey, paramValue, comment) -def process_cloud_definition(cloudPassportYaml, env_dir, comment) : +def process_cloud_definition(cloudPassportYaml, env_dir, comment, isExternalCredEnv) : cloud_schema="schemas/cloud.schema.json" # cloud cloudYamlPath = f"{env_dir}/cloud.yml" @@ -41,7 +43,7 @@ def process_cloud_definition(cloudPassportYaml, env_dir, comment) : maasPassportYaml = cloudPassportYaml["maas"] # if maas is presented in cloud passport - than maas is enabled store_value_to_yaml(cloudYaml["maasConfig"], "enable", True, comment) - store_value_to_yaml(cloudYaml["maasConfig"], "credentialsId", get_cred_id_from_cred_macros(maasPassportYaml["MAAS_CREDENTIALS_USERNAME"]), comment) + store_value_to_yaml(cloudYaml["maasConfig"], "credentialsId", extract_cred_id(maasPassportYaml["MAAS_CREDENTIALS_USERNAME"], isExternalCredEnv), comment) process_and_update_key("maasUrl", cloudYaml["maasConfig"], "MAAS_SERVICE_ADDRESS", maasPassportYaml, comment) process_and_update_key("maasInternalAddress", cloudYaml["maasConfig"], "MAAS_INTERNAL_ADDRESS", maasPassportYaml, comment) del cloudPassportYaml["maas"] @@ -51,7 +53,7 @@ def process_cloud_definition(cloudPassportYaml, env_dir, comment) : vaultPassportYaml = cloudPassportYaml["vault"] vaultUrl = vaultPassportYaml["VAULT_ADDR"] if vaultUrl : - store_value_to_yaml(cloudYaml["vaultConfig"], "credentialsId", get_cred_id_from_cred_macros(vaultPassportYaml["VAULT_AUTH_ROLE_ID"]), comment) + store_value_to_yaml(cloudYaml["vaultConfig"], "credentialsId", extract_cred_id(vaultPassportYaml["VAULT_AUTH_ROLE_ID"], isExternalCredEnv), comment) store_value_to_yaml(cloudYaml["vaultConfig"], "enable", True, comment) store_value_to_yaml(cloudYaml["vaultConfig"], "url", vaultUrl, comment) else : @@ -67,7 +69,7 @@ def process_cloud_definition(cloudPassportYaml, env_dir, comment) : cloudYaml["dbaasConfigs"] = yaml.load("[]") cloudYaml["dbaasConfigs"].append(yaml.load("enable: false")) dbaasConfigYaml = cloudYaml["dbaasConfigs"][0] - store_value_to_yaml(dbaasConfigYaml, "credentialsId", get_cred_id_from_cred_macros(dbaasPassportYaml["DBAAS_CLUSTER_DBA_CREDENTIALS_USERNAME"]), comment) + store_value_to_yaml(dbaasConfigYaml, "credentialsId", extract_cred_id(dbaasPassportYaml["DBAAS_CLUSTER_DBA_CREDENTIALS_USERNAME"], isExternalCredEnv), comment) store_value_to_yaml(dbaasConfigYaml, "enable", True, comment) process_and_update_key("apiUrl", dbaasConfigYaml, "API_DBAAS_ADDRESS", dbaasPassportYaml, comment) process_and_update_key("aggregatorUrl", dbaasConfigYaml, "DBAAS_AGGREGATOR_ADDRESS", dbaasPassportYaml, comment) @@ -78,7 +80,7 @@ def process_cloud_definition(cloudPassportYaml, env_dir, comment) : if "consul" in cloudPassportYaml : consulPassportYaml = cloudPassportYaml["consul"] consulConfigYaml = cloudYaml["consulConfig"] - store_value_to_yaml(consulConfigYaml, "tokenSecret", get_cred_id_from_cred_macros(consulPassportYaml["CONSUL_ADMIN_TOKEN"]), comment) + store_value_to_yaml(consulConfigYaml, "tokenSecret", extract_cred_id(consulPassportYaml["CONSUL_ADMIN_TOKEN"], isExternalCredEnv), comment) process_and_update_key("enabled", consulConfigYaml, "CONSUL_ENABLED", consulPassportYaml, comment) process_and_update_key("publicUrl", consulConfigYaml, "CONSUL_PUBLIC_URL", consulPassportYaml, comment) process_and_update_key("internalUrl", consulConfigYaml, "CONSUL_URL", consulPassportYaml, comment) @@ -87,14 +89,15 @@ def process_cloud_definition(cloudPassportYaml, env_dir, comment) : del cloudPassportYaml["consul"] else: store_value_to_yaml(cloudYaml["consulConfig"], "enabled", False) + # adding rest of cloud passport parameters to cloud deploy parameters logger.debug(f"Rest of params from cloud passport are: \n{dump_as_yaml_format(cloudPassportYaml)}") - mergeDeployParametersFromPassport(cloudPassportYaml, cloudYaml, comment) + mergeDeployParametersFromPassport(cloudPassportYaml, cloudYaml, comment, isExternalCredEnv) # storing cloud yaml writeYamlToFile(cloudYamlPath, cloudYaml) beautifyYaml(cloudYamlPath, cloud_schema) -def add_cloud_passport_creds(cloud_passport_name, cloud_passport_file_path, env_dir, comment): +def add_cloud_passport_creds(cloud_passport_name, cloud_passport_file_path, env_dir, comment, isExternalCredEnv): logger.info(f"Searching credentials for cloud passport {cloud_passport_file_path}") credsSchema="schemas/credential.schema.json" # first searching in subfolder @@ -111,16 +114,19 @@ def add_cloud_passport_creds(cloud_passport_name, cloud_passport_file_path, env_ else: logger.error(f"No cloud pasport credentials files found in either {passportSubfolderPath} or {passportSameFolderPath}.") raise ReferenceError(f"No cloud pasport credentials files found. See logs above") - envCredentialsPath = f"{env_dir}/Credentials/credentials.yml" - if os.path.exists(envCredentialsPath) : - envCredsYaml = openYaml(envCredentialsPath) - else: - envCredsYaml = yaml.load("{}") - for key, value in passportCredsYaml.items() : - store_value_to_yaml(envCredsYaml, key, value, comment) - # storing credentials yaml - writeYamlToFile(envCredentialsPath, envCredsYaml) - beautifyYaml(envCredentialsPath, credsSchema) + copy_creds_to_env_creds_file(env_dir, passportCredsYaml, comment, credsSchema) + +def extract_cred_id(param, isExternal=False): + if isExternal: + if isinstance(param, dict): + cred_id = extract_external_cred(param) + if cred_id is None: + raise ValueError(f"Invalid external cred: expected '$type'='credRef' with valid 'credId' in {param} in cloud passport") + return cred_id + else: + return get_cred_id_from_cred_macros(param) + return get_cred_id_from_cred_macros(param) + def update_env_definition_with_cloud_name(render_env_dir, source_env_dir, all_instances_dir): inventoryYaml = getEnvDefinition(render_env_dir) @@ -132,7 +138,7 @@ def update_env_definition_with_cloud_name(render_env_dir, source_env_dir, all_in else: logger.info(f"No cloud name found for env {render_env_dir} from passport") -def process_cloud_passport(render_env_dir, env_instances_dir, instances_dir) : +def process_cloud_passport(render_env_dir, env_instances_dir, instances_dir, isExternalCredEnv=False) : logger.info(f"Trying to find cloud passport definition file") cloudPassportFilePath = find_cloud_passport_definition(env_instances_dir, instances_dir) cloudPassportFileName = extractNameFromFile(cloudPassportFilePath) @@ -142,7 +148,7 @@ def process_cloud_passport(render_env_dir, env_instances_dir, instances_dir) : logger.info(f"Processing cloud passport: {cloudPassportFilePath}") cloudPassportYaml = openYaml(cloudPassportFilePath) comment = f"cloud passport: {cloudPassportFileName} version: {cloudPassportYaml['version']}" - process_cloud_definition(cloudPassportYaml, render_env_dir, comment) - add_cloud_passport_creds(cloudPassportFileName, cloudPassportFilePath, render_env_dir, comment) + process_cloud_definition(cloudPassportYaml, render_env_dir, comment, isExternalCredEnv) + add_cloud_passport_creds(cloudPassportFileName, cloudPassportFilePath, render_env_dir, comment, isExternalCredEnv) else: logger.info("No cloud passport definition found. Cloud passport processing skipped...") diff --git a/scripts/build_env/create_credentials.py b/scripts/build_env/create_credentials.py index 441e2a550..d704f0411 100644 --- a/scripts/build_env/create_credentials.py +++ b/scripts/build_env/create_credentials.py @@ -1,11 +1,13 @@ from pathlib import Path import os from envgenehelper import * +from typing import Optional, Set #const CRED_TYPE_SECRET="secret" CRED_TYPE_USERPASS="usernamePassword" CRED_TYPE_VAULT="vaultAppRole" +CRED_TYPE_EXTERNAL="external" def createCredDefinition(credId, credType) : cred = {} @@ -13,27 +15,34 @@ def createCredDefinition(credId, credType) : cred["type"] = credType return cred -def processParametersAndAppend(paramTypeKey, paramsDict, credsList, tenantName, cloudName="", namespaceName="", comment="") : +def processParametersAndAppend(paramTypeKey, paramsDict, credsList, tenantName, cloudName="", namespaceName="", comment="", externalCredIds=None) : if paramTypeKey not in paramsDict.keys(): return - processDictAndAppend(paramsDict[paramTypeKey], credsList, tenantName, cloudName, namespaceName, comment) + processDictAndAppend(paramsDict[paramTypeKey], credsList, tenantName, cloudName, namespaceName, comment, externalCredIds) -def processDictAndAppend(params, credsList, tenantName, cloudName, namespaceName, comment): +def processDictAndAppend(params, credsList, tenantName, cloudName, namespaceName, comment, externalCredIds=None): for key, value in params.items(): - processSingleParam(key, value, credsList, tenantName, cloudName, namespaceName, comment) + processSingleParam(key, value, credsList, tenantName, cloudName, namespaceName, comment, externalCredIds) -def processSingleParam(key, value, credsList, tenantName, cloudName, namespaceName, comment): +def processSingleParam(key, value, credsList, tenantName, cloudName, namespaceName, comment, externalCredIds: Optional[Set[str]] = None): if isinstance(value, dict): - processDictAndAppend(value, credsList, tenantName, cloudName, namespaceName, comment) + cred_id = extract_external_cred(value) + if cred_id and externalCredIds is not None: + externalCredIds.add(cred_id) + return + processDictAndAppend(value, credsList, tenantName, cloudName, namespaceName, comment, externalCredIds) elif isinstance(value, list): # if is array, than iterate for idx, item in enumerate(value): - value[idx] = processSingleParam(idx, item, credsList, tenantName, cloudName, namespaceName, comment) + value[idx] = processSingleParam(idx, item, credsList, tenantName, cloudName, namespaceName, comment, externalCredIds) elif isinstance(value, str): if check_is_cred(key, value): appendCredList(get_cred_list_from_param(key, value, True, tenantName, cloudName, namespaceName), credsList, comment) -def checkCredAndAppend(credName, credsList, secretType, comment=""): +def checkCredAndAppend(credName, credsList, secretType, comment="", isExternalCredEnv=False, externalCredIds=None): if (credName): + if isExternalCredEnv and externalCredIds is not None: + externalCredIds.add(credName) + return appendCredList([createCredDefinition(credName, secretType)], credsList, comment) return credsList @@ -44,27 +53,49 @@ def appendCredList(additionalCreds, wholeCredsList, comment=""): credMeta["comment"] = comment wholeCredsList.append(credMeta) -def getTenantCreds(tenantContent, tenantName): +def getTenantCreds(tenantContent, tenantName, isExternalCredEnv=False, externalCredIds=None): creds = [] tenantComment = f"tenant {tenantName}" - checkCredAndAppend(tenantContent["credential"], creds, CRED_TYPE_SECRET, tenantComment) + checkCredAndAppend(tenantContent["credential"], creds, CRED_TYPE_SECRET, tenantComment, isExternalCredEnv, externalCredIds) #process deployParameters - processParametersAndAppend("deployParameters", tenantContent, creds, tenantName, comment=tenantComment) + processParametersAndAppend("deployParameters", tenantContent, creds, tenantName, comment=tenantComment, externalCredIds=externalCredIds) processParametersAndAppend("environmentParameters", tenantContent["globalE2EParameters"], creds, tenantName, comment=tenantComment) return creds -def getCloudCreds(cloudContent, tenantName, cloudName): +def validateExternalCreds(envCredsMap, extCredIds): + logger.info(f"External cred ids found across entities are {extCredIds}") + notFoundCred = [ + credName for credName in extCredIds + if credName not in envCredsMap + ] + if notFoundCred: + raise ValueError( + f"Following external credentials:\n {notFoundCred}\n referred in environment are not found in any external credential source") + orphanCreds = [ + credName + for credName, credConfig in envCredsMap.items() + if credConfig.get("type") == "external" + and credName not in extCredIds + ] + if orphanCreds: + logger.warning(f"Following external credentials:\n{orphanCreds}\n" + f"exist in external credential source but are not referred in environment" + ) + logger.info(f'{len(extCredIds)} external creds processed from environment') + + +def getCloudCreds(cloudContent, tenantName, cloudName, isExternalCredEnv=False, externalCredIds=None): creds = [] cloudComment = f"cloud {cloudName}" - checkCredAndAppend(cloudContent["defaultCredentialsId"], creds, CRED_TYPE_SECRET, cloudComment) - checkCredAndAppend(cloudContent["maasConfig"]["credentialsId"], creds, CRED_TYPE_USERPASS, cloudComment) - checkCredAndAppend(cloudContent["vaultConfig"]["credentialsId"], creds, CRED_TYPE_SECRET, cloudComment) - checkCredAndAppend(cloudContent["consulConfig"]["tokenSecret"], creds, CRED_TYPE_SECRET, cloudComment) + checkCredAndAppend(cloudContent["defaultCredentialsId"], creds, CRED_TYPE_SECRET, cloudComment, isExternalCredEnv, externalCredIds) + checkCredAndAppend(cloudContent["maasConfig"]["credentialsId"], creds, CRED_TYPE_USERPASS, cloudComment, isExternalCredEnv, externalCredIds) + checkCredAndAppend(cloudContent["vaultConfig"]["credentialsId"], creds, CRED_TYPE_SECRET, cloudComment, isExternalCredEnv, externalCredIds) + checkCredAndAppend(cloudContent["consulConfig"]["tokenSecret"], creds, CRED_TYPE_SECRET, cloudComment, isExternalCredEnv, externalCredIds) for i in cloudContent["dbaasConfigs"]: - checkCredAndAppend(i["credentialsId"], creds, CRED_TYPE_USERPASS, cloudComment) + checkCredAndAppend(i["credentialsId"], creds, CRED_TYPE_USERPASS, cloudComment, isExternalCredEnv, externalCredIds) #process deployParameters - processParametersAndAppend("deployParameters", cloudContent, creds, tenantName, cloudName, comment=cloudComment) + processParametersAndAppend("deployParameters", cloudContent, creds, tenantName, cloudName, comment=cloudComment, externalCredIds=externalCredIds) #process e2eParameters processParametersAndAppend("e2eParameters", cloudContent, creds, tenantName, cloudName, comment=cloudComment) #process technicalConfigurationParameters @@ -72,25 +103,25 @@ def getCloudCreds(cloudContent, tenantName, cloudName): return creds -def get_bg_domain_creds(content, name): +def get_bg_domain_creds(content, name, isExternalCredEnv=False, externalCredIds=None): creds = [] bg_domain_comment = f"bg domain {name}" - checkCredAndAppend(content["controllerNamespace"]["credentials"], creds, CRED_TYPE_SECRET, bg_domain_comment) + checkCredAndAppend(content["controllerNamespace"]["credentials"], creds, CRED_TYPE_SECRET, bg_domain_comment, isExternalCredEnv, externalCredIds) return creds -def getNamespaceCreds(namespaceContent, tenantName, cloudName, namespaceName): +def getNamespaceCreds(namespaceContent, tenantName, cloudName, namespaceName, isExternalCredEnv=False, externalCredIds=None): creds = [] namespaceComment = f"namespace {namespaceName}" - checkCredAndAppend(namespaceContent["credentialsId"], creds, CRED_TYPE_SECRET, namespaceComment) + checkCredAndAppend(namespaceContent["credentialsId"], creds, CRED_TYPE_SECRET, namespaceComment, isExternalCredEnv, externalCredIds) #process deployParameters - processParametersAndAppend("deployParameters", namespaceContent, creds, tenantName, cloudName, namespaceName, comment=namespaceComment) + processParametersAndAppend("deployParameters", namespaceContent, creds, tenantName, cloudName, namespaceName, comment=namespaceComment, externalCredIds=externalCredIds) #process e2eParameters processParametersAndAppend("e2eParameters", namespaceContent, creds, tenantName, cloudName, namespaceName, comment=namespaceComment) #process technicalConfigurationParameters processParametersAndAppend("technicalConfigurationParameters", namespaceContent, creds, tenantName, cloudName, namespaceName, comment=namespaceComment) return creds -def getApplicationCreds(appPath, tenantName, cloudName, namespaceName=""): +def getApplicationCreds(appPath, tenantName, cloudName, namespaceName="", externalCredIds=None): creds = [] appContent = openYaml(appPath) appName = appContent["name"] @@ -99,7 +130,7 @@ def getApplicationCreds(appPath, tenantName, cloudName, namespaceName=""): else: comment = f"cloud {cloudName} application {appName}" #process deployParameters - processParametersAndAppend("deployParameters", appContent, creds, tenantName, cloudName, namespaceName, comment=comment) + processParametersAndAppend("deployParameters", appContent, creds, tenantName, cloudName, namespaceName, comment=comment, externalCredIds=externalCredIds) #process technicalConfigurationParameters processParametersAndAppend("technicalConfigurationParameters", appContent, creds, tenantName, cloudName, namespaceName, comment=comment) return creds @@ -194,18 +225,20 @@ def mergeSharedCreds(credYamlPath, envDir, instancesDir) : count += 1 logger.info(f"Added {count} shared master credentials from {credFilePath}") writeYamlToFile(credYamlPath, credsYaml) + return credsYaml -def create_credentials(envDir, envInstancesDir, instancesDir) : +def create_credentials(envDir, envInstancesDir, instancesDir, isExternalCredEnv) : logger.info(f"Start to create credentials: envDir={envDir}, envInstancesDir={envInstancesDir}, instancesDir={instancesDir}") logger.info(f"Creating credentials for environment directory: {envDir}") credsSchema="schemas/credential.schema.json" resultingCreds = [] #tenant tenantFileName = envDir+"/tenant.yml" + externalCredIds = set() logger.info(f"Processing tenant") tenantYaml = openYaml(tenantFileName) tenantName = tenantYaml["name"] - mergeResult = mergeCreds(getTenantCreds(tenantYaml, tenantName), resultingCreds) + mergeResult = mergeCreds(getTenantCreds(tenantYaml, tenantName, isExternalCredEnv, externalCredIds), resultingCreds) logger.info(f'{mergeResult["countAdded"]} creds added from tenant {tenantFileName}') resultingCreds = mergeResult["mergedCreds"] #cloud @@ -213,7 +246,7 @@ def create_credentials(envDir, envInstancesDir, instancesDir) : logger.info(f"Processing cloud") cloudYaml = openYaml(cloudFileName) cloudName = cloudYaml["name"] - mergeResult = mergeCreds(getCloudCreds(cloudYaml, tenantName, cloudName), resultingCreds) + mergeResult = mergeCreds(getCloudCreds(cloudYaml, tenantName, cloudName, isExternalCredEnv, externalCredIds), resultingCreds) logger.info(f'{mergeResult["countAdded"]} creds added from cloud {cloudFileName}') resultingCreds = mergeResult["mergedCreds"] #bgd object @@ -230,7 +263,7 @@ def create_credentials(envDir, envInstancesDir, instancesDir) : # iterate through cloud applications and create cred definitions applications = findAllYamlsInDir(f"{envDir}/Applications") for appPath in applications : - mergeResult = mergeCreds(getApplicationCreds(appPath, tenantName, cloudName), resultingCreds) + mergeResult = mergeCreds(getApplicationCreds(appPath, tenantName, cloudName, externalCredIds=externalCredIds), resultingCreds) logger.info(f'{mergeResult["countAdded"]} creds added for cloud application {appPath}') resultingCreds = mergeResult["mergedCreds"] # iterate through namespaces and create cred definitions @@ -242,7 +275,7 @@ def create_credentials(envDir, envInstancesDir, instancesDir) : namespaceKey = extract_namespace_from_namespace_path(namespacePath) namespaceName = namespaceYaml["name"] namespaceNameMap[namespaceKey] = namespaceName - mergeResult = mergeCreds(getNamespaceCreds(namespaceYaml, tenantName, cloudName, namespaceName), resultingCreds) + mergeResult = mergeCreds(getNamespaceCreds(namespaceYaml, tenantName, cloudName, namespaceName, isExternalCredEnv, externalCredIds), resultingCreds) logger.info(f'{mergeResult["countAdded"]} creds added for namespace {namespacePath}') resultingCreds = mergeResult["mergedCreds"] # iterate through namespace applications and create cred definitions @@ -251,13 +284,29 @@ def create_credentials(envDir, envInstancesDir, instancesDir) : for appPath in applications : namespaceKey = extract_namespace_from_application_path(appPath) namespaceName = namespaceNameMap[namespaceKey] - mergeResult = mergeCreds(getApplicationCreds(appPath, tenantName, cloudName, namespaceName), resultingCreds) + mergeResult = mergeCreds(getApplicationCreds(appPath, tenantName, cloudName, namespaceName, externalCredIds), resultingCreds) logger.info(f'{mergeResult["countAdded"]} creds added for namespace application {appPath}') resultingCreds = mergeResult["mergedCreds"] - # store credentials + + #store credentials credYamlPath = envDir + "/Credentials/credentials.yml" mergeAndSaveYaml(credYamlPath, resultingCreds) # process shared credentials - mergeSharedCreds(credYamlPath, envInstancesDir, instancesDir) - beautifyYaml(credYamlPath, credsSchema) - + envCredsMap = mergeSharedCreds(credYamlPath, envInstancesDir, instancesDir) + #validate external credentials + if isExternalCredEnv: + if resultingCreds: + #to cover condition like local creds macro with external cred id + local_cred_ids = [ + item.get('cred', {}).get('credentialsId') + for item in resultingCreds + if item.get('cred', {}).get('credentialsId') + ] + raise ReferenceError(f"Found local credential macros in external cred only environment. Credential IDs are {local_cred_ids}") + logger.info(f"Validating external credentials for external only environment") + validateExternalCreds(envCredsMap, externalCredIds) + else: + if externalCredIds: + raise ReferenceError(f"Found external credential references in parameters in local cred only environment. Credential IDs are {externalCredIds}") + validate_cred_types(envCredsMap, isExternalCredEnv, credYamlPath) + beautifyYaml(credYamlPath, credsSchema) \ No newline at end of file diff --git a/scripts/build_env/main.py b/scripts/build_env/main.py index fb12545a0..e6b5ba9f9 100644 --- a/scripts/build_env/main.py +++ b/scripts/build_env/main.py @@ -108,6 +108,7 @@ def build_environment(env_name, cluster_name, templates_dirs, source_env_dir, al render_parameters_dir = f"{base_dir}/tmp/parameters_templates" render_profiles_dir = f"{base_dir}/tmp/resource_profiles" + namespaces_path = get_namespaces_path() if check_dir_exists(str(namespaces_path.absolute())): logger.info("Namespaces found, saving them into tmp location") @@ -175,17 +176,18 @@ def build_environment(env_name, cluster_name, templates_dirs, source_env_dir, al envvars["cmdb_url"] = cmdb_url envvars["output_dir"] = output_dir envvars["render_profiles_dir"] = render_profiles_dir + envvars["work_dir"] = str(work_dir) render_context = EnvGenerator() render_context.render_config_env(env_name, envvars) handle_template_override(render_dir) env_specific_resource_profile_map = get_env_specific_resource_profiles(source_env_dir, all_instances_dir, ENV_SPECIFIC_RESOURCE_PROFILE_SCHEMA) build_env(env_name, source_env_dir, render_parameters_dir, render_dir, render_profiles_dir, - env_specific_resource_profile_map, all_instances_dir, render_context, templates_dirs) + env_specific_resource_profile_map, all_instances_dir, render_context, templates_dirs, render_context.isExternalCredEnv) resulting_dir = post_process_env_after_rendering(env_name, render_env_dir, source_env_dir, all_instances_dir, output_dir) - - return resulting_dir + logger.info(f"External cred env is set as {render_context.isExternalCredEnv}") + return resulting_dir, render_context.isExternalCredEnv def get_duplicate_names(param_files): @@ -286,9 +288,9 @@ def render_environment(env_name, cluster_name, templates_dirs, all_instances_dir env_dir = get_env_instances_dir(env_name, cluster_name, all_instances_dir) logger.info(f"Environment {env_name} directory is {env_dir}") - resulting_env_dir = build_environment(env_name, cluster_name, templates_dirs, env_dir, all_instances_dir, + resulting_env_dir, isExternalCredEnv = build_environment(env_name, cluster_name, templates_dirs, env_dir, all_instances_dir, output_dir, work_dir) - create_credentials(resulting_env_dir, env_dir, all_instances_dir) + create_credentials(resulting_env_dir, env_dir, all_instances_dir, isExternalCredEnv) apply_ns_build_filter() diff --git a/scripts/build_env/render_config_env.py b/scripts/build_env/render_config_env.py index 8ea5383ef..a973b4f02 100644 --- a/scripts/build_env/render_config_env.py +++ b/scripts/build_env/render_config_env.py @@ -15,7 +15,10 @@ SCHEMAS_DIR = Path(__file__).resolve().parents[2] / "schemas" APPDEF_SCHEMA = str(SCHEMAS_DIR / "appdef.schema.json") +SECRET_SCHEMA = str(SCHEMAS_DIR / "secret-stores.schema.json") +CRED_SCHEMA = str(SCHEMAS_DIR / "credential.schema.json") TD_SCHEMA = str(SCHEMAS_DIR / "template-descriptor.schema.json") +EXTERNAL_CRED_COMMENT="external credential template" yml = create_yaml_processor() @@ -49,6 +52,7 @@ class Context(BaseModel): render_parameters_dir: Optional[str] = '' env_vars: OrderedDict = Field(default_factory=OrderedDict) render_profiles_dir: Optional[str] = '' + work_dir: Optional[str] = '' start_time: datetime | None = Field(default=None, exclude=True) end_time: datetime | None = Field(default=None, exclude=True) @@ -91,6 +95,7 @@ def render_obj_by_context(template: dict, context: Context) -> dict: class EnvGenerator: def __init__(self): self.ctx = Context() + self.isExternalCredEnv = False logger.debug("EnvGenerator initialized with context: %s", self.ctx.dict(exclude_none=True, exclude={"env_vars"})) @@ -427,6 +432,30 @@ def generate_composite_structure(self): cs_file = Path(current_env_dir) / "composite_structure.yml" cs_file.parent.mkdir(parents=True, exist_ok=True) self.render_from_file_to_file(Template(composite_structure).render(self.ctx.as_dict()), str(cs_file)) + + def generate_external_cred(self): + #render external creds + external_credential_template = self.ctx.current_env_template.get("external_credential_template") + if not external_credential_template: + return + external_cred_path = Template(external_credential_template).render(self.ctx.as_dict()) + logger.info(f"Found external template. Render external credentials for {external_cred_path}") + external_creds = openYaml(external_cred_path) + default_remote_path = "{{ current_env.cloud }}/{{ current_env.name }}" + for credConfig in external_creds.values(): + if isinstance(credConfig, dict) and "remoteRefPath" not in credConfig: + credConfig["remoteRefPath"] = default_remote_path + rendered_external_creds = render_obj_by_context(external_creds, self.ctx) + logger.debug(f"Rendered external credentials is: \n{rendered_external_creds}") + + #validate secret file + secretStoreFile = f"{self.ctx.work_dir}/configuration/secret-stores.yml" + secretStoreMap = openYaml(secretStoreFile) + validateSchema(secretStoreMap, schema_path=SECRET_SCHEMA) + + #copy rendred creds to env creds file seperately to propagate comments + copy_creds_to_env_creds_file(self.ctx.current_env_dir, rendered_external_creds, EXTERNAL_CRED_COMMENT, CRED_SCHEMA) + self.isExternalCredEnv = True def get_rendered_target_path(self, template_path: Path) -> Path: path_str = str(template_path) @@ -628,6 +657,7 @@ def render_config_env(self, env_name: str, extra_env: dict): self.generate_cloud_file() self.generate_namespace_files() self.generate_composite_structure() + self.generate_external_cred() env_specific_schema = self.ctx.current_env_template.get("envSpecificSchema") if env_specific_schema: diff --git a/scripts/build_env/tests/env-build/test_render_envs.py b/scripts/build_env/tests/env-build/test_render_envs.py index 6f165b78a..ae7c962d5 100644 --- a/scripts/build_env/tests/env-build/test_render_envs.py +++ b/scripts/build_env/tests/env-build/test_render_envs.py @@ -21,6 +21,7 @@ ("bgd-cluster", "bgd-env", "bgd", {}), ("bgd-cluster", "bgd-ns-artifacts-env", "bgd-ns-artifacts", {NamespaceRole.PEER: "test_data/test_templates_peer", NamespaceRole.ORIGIN: "test_data/test_templates_origin"}), ("cluster03", "rpo-replacement-mode", "simple", {}), + ("cluster-01", "env-extcred", "extcred-template", {}), ] diff --git a/test_data/configuration/secret-stores.yml b/test_data/configuration/secret-stores.yml new file mode 100644 index 000000000..cad0b449a --- /dev/null +++ b/test_data/configuration/secret-stores.yml @@ -0,0 +1,16 @@ +default-store: + type: vault + url: https://my-keyvault.vault.azure.net/ + mountPath: secret/data/app +custom-store: + type: azure + url: https://azure.mycompany.com + vaultName: my-keyvault +maas-store: + type: aws + url: https://my-aws.nc.net/ + region: India +dbaas-store: + type: gcp + url: https://my-gcp.nc.net/ + projectId: agf56hoji8 diff --git a/test_data/test_environments/cluster-01/cloud-passport/ext-mode-creds.yml b/test_data/test_environments/cluster-01/cloud-passport/ext-mode-creds.yml new file mode 100644 index 000000000..eead91271 --- /dev/null +++ b/test_data/test_environments/cluster-01/cloud-passport/ext-mode-creds.yml @@ -0,0 +1,18 @@ +app-dbaas-cred: + type: "external" + secretStore: "azure-store" + create: false + properties: + - name: "username" + - name: "password" +app-maas-cred: + type: "external" + secretStore: "azure-store" + create: true + properties: + - name: "username" + - name: "password" +app-sidecar-token: + type: "external" + secretStore: "vault-store" + remoteRefPath: "{{ current_env.cloud }}/{{ current_env.name }}/envgene" diff --git a/test_data/test_environments/cluster-01/cloud-passport/ext-mode.yml b/test_data/test_environments/cluster-01/cloud-passport/ext-mode.yml new file mode 100644 index 000000000..e4848fe19 --- /dev/null +++ b/test_data/test_environments/cluster-01/cloud-passport/ext-mode.yml @@ -0,0 +1,60 @@ +--- +version: 1.5 +cloud: + CLOUD_API_HOST: api.cluster-01.qubership.org + CLOUD_API_PORT: "6443" + CLOUD_DEPLOY_TOKEN: app-sidecar-token + CLOUD_PUBLIC_HOST: cluster-01.qubership.org + CLOUD_PRIVATE_HOST: cluster-01.qubership.org + CLOUD_DASHBOARD_URL: https://dashboard.cluster-01.qubership.org + CLOUD_PROTOCOL: https + PRODUCTION_MODE: false + GRAYLOG_UI_URL: https://cluster-01.qubership.org + TRACING_UI_URL: https://cluster-01.qubership.org + GRAFANA_UI_URL: https://cluster-01.qubership.org + CMDB_URL: https://cluster-01.qubership.org +dbaas: + API_DBAAS_ADDRESS: http://dbaas.dbaas:8080 + DBAAS_AGGREGATOR_ADDRESS: https://dbaas.cluster-01.qubership.org + DBAAS_CLUSTER_DBA_CREDENTIALS_USERNAME: + $type: "credRef" + credId: "app-dbaas-cred" + property: "username" + DBAAS_CLUSTER_DBA_CREDENTIALS_PASSWORD: + $type: "credRef" + credId: "app-dbaas-cred" + property: "password" +maas: + MAAS_INTERNAL_ADDRESS: http://maas.maas:8080 + MAAS_SERVICE_ADDRESS: http://maas.cluster-01.qubership.org + MAAS_CREDENTIALS_USERNAME: ${creds.get("app-maas-cred").username} + MAAS_CREDENTIALS_PASSWORD: ${creds.get("app-maas-cred").password} +consul: + CONSUL_URL: http://consul.consul:8080 + CONSUL_ENABLED: true + CONSUL_PUBLIC_URL: http://consul.consul:8080 + CONSUL_ADMIN_TOKEN: ${creds.get("app-sidecar-token").secret} +zookeeper: + ZOOKEEPER_URL: ${ZOOKEEPER_ADDRESS} + ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 +storage: + STORAGE_SERVER_URL: https://minio.cluster-01.qubership.org + STORAGE_PROVIDER: s3 + STORAGE_REGION: eu-west-1 + STORAGE_RWO_CLASS: standard + STORAGE_RWX_CLASS: "" +core: + DEFAULT_TENANT_NAME: tenant + DEFAULT_TENANT_ADMIN_LOGIN: admin + DEFAULT_TENANT_ADMIN_PASSWORD: password + MAVEN_REPO_URL: https://artifactory.qubership.org + MAVEN_REPO_NAME: mvn.group + MAVEN_REPO_STAGING_NAME: ${MAVEN_REPO_NAME} + MAVEN_REPO_DEV_NAME: ${MAVEN_REPO_NAME} +global: + MONITORING_ENABLED: "true" + TRACING_ENABLED: "false" + TRACING_HOST: tracing-agent +bss: + DOC_STORAGE_TEMPORARY_BUCKET_NAME: temporary-bucket + DOC_STORAGE_PERSISTENT_BUCKET_NAME: permanent-bucket \ No newline at end of file diff --git a/test_data/test_environments/cluster-01/credentials/external-ci-creds.yml b/test_data/test_environments/cluster-01/credentials/external-ci-creds.yml new file mode 100644 index 000000000..44a669c73 --- /dev/null +++ b/test_data/test_environments/cluster-01/credentials/external-ci-creds.yml @@ -0,0 +1,8 @@ +shared-cred: + type: "external" + secretStore: "maas-store" + remoteRefPath: "cluster-01/env-ext-cred" + create: false + properties: + - name: "username" + - name: "password" \ No newline at end of file diff --git a/test_data/test_environments/cluster-01/env-extcred/Credentials/credentials.yml b/test_data/test_environments/cluster-01/env-extcred/Credentials/credentials.yml new file mode 100644 index 000000000..201313f82 --- /dev/null +++ b/test_data/test_environments/cluster-01/env-extcred/Credentials/credentials.yml @@ -0,0 +1,42 @@ +app-db-cred: # external credential template + type: "external" + secretStore: "azure-store" + remoteRefPath: "cluster_01_env_extcred/env-extcred" + create: false + properties: + - name: "username" + - name: "password" +app-dbaas-cred: # cloud passport: ext-mode version: 1.5 + type: "external" + secretStore: "azure-store" + create: false + properties: + - name: "username" + - name: "password" +app-maas-cred: # cloud passport: ext-mode version: 1.5 + type: "external" + secretStore: "azure-store" + create: true + properties: + - name: "username" + - name: "password" +app-sidecar-token: # cloud passport: ext-mode version: 1.5 + type: "external" + secretStore: "vault-store" + remoteRefPath: "{{ current_env.cloud }}/{{ current_env.name }}/envgene" +app-test-cred: # external credential template + type: "external" + secretStore: "gcp-store" + remoteRefPath: "cluster_01_env_extcred/env-extcred/envgene" + create: true + properties: + - name: "username" + - name: "password" +shared-cred: # shared credentials: external-ci-creds + type: "external" + secretStore: "maas-store" + remoteRefPath: "cluster-01/env-ext-cred" + create: false + properties: + - name: "username" + - name: "password" diff --git a/test_data/test_environments/cluster-01/env-extcred/Inventory/env_definition.yml b/test_data/test_environments/cluster-01/env-extcred/Inventory/env_definition.yml new file mode 100644 index 000000000..8e8f4e3d4 --- /dev/null +++ b/test_data/test_environments/cluster-01/env-extcred/Inventory/env_definition.yml @@ -0,0 +1,21 @@ +inventory: + environmentName: env-extcred + tenantName: tenant + cloudName: cluster_01_env_extcred + cloudPassport: ext-mode + description: External Cred Sample + owners: Qubership team +envTemplate: + name: "extcred-template" + artifact: "deployment-configuration-env-templates:1.2.3" + additionalTemplateVariables: + site: "offsite" + use_env_prefix: true + sharedMasterCredentialFiles: + - "external-ci-creds" + envSpecificParamsets: + app: + - "extcred" + envSpecificE2EParamsets: {} +generatedVersions: + generateEnvironmentLatestVersion: "deployment-configuration-env-templates:extcred-template" # This value is automatically generated during job run. diff --git a/test_data/test_environments/cluster-01/env-extcred/Namespaces/app/Applications/eso-app.yml b/test_data/test_environments/cluster-01/env-extcred/Namespaces/app/Applications/eso-app.yml new file mode 100644 index 000000000..7415e3865 --- /dev/null +++ b/test_data/test_environments/cluster-01/env-extcred/Namespaces/app/Applications/eso-app.yml @@ -0,0 +1,8 @@ +# The contents of this file is generated from template artifact: deployment-configuration-env-templates. +# Contents will be overwritten by next generation. +# Please modify this contents only for development purposes or as workaround. +name: "eso-app" +deployParameters: + SECRET_FLOW: "external-values" # paramset: extcred version: 1 source: template + TEST_NAMESPACE: "apps-test" # paramset: extcred version: 1 source: template +technicalConfigurationParameters: {} diff --git a/test_data/test_environments/cluster-01/env-extcred/Namespaces/app/Applications/vals-app.yml b/test_data/test_environments/cluster-01/env-extcred/Namespaces/app/Applications/vals-app.yml new file mode 100644 index 000000000..acf1bcbb9 --- /dev/null +++ b/test_data/test_environments/cluster-01/env-extcred/Namespaces/app/Applications/vals-app.yml @@ -0,0 +1,8 @@ +# The contents of this file is generated from template artifact: deployment-configuration-env-templates. +# Contents will be overwritten by next generation. +# Please modify this contents only for development purposes or as workaround. +name: "vals-app" +deployParameters: + INTEGRATED_SYSTEMS: "ABC" # paramset: extcred version: 1 source: template + TEST_NAMESPACE: "apps-test" # paramset: extcred version: 1 source: template +technicalConfigurationParameters: {} diff --git a/test_data/test_environments/cluster-01/env-extcred/Namespaces/app/namespace.yml b/test_data/test_environments/cluster-01/env-extcred/Namespaces/app/namespace.yml new file mode 100644 index 000000000..cf46a4af8 --- /dev/null +++ b/test_data/test_environments/cluster-01/env-extcred/Namespaces/app/namespace.yml @@ -0,0 +1,36 @@ +# The contents of this file is generated from template artifact: deployment-configuration-env-templates. +# Contents will be overwritten by next generation. +# Please modify this contents only for development purposes or as workaround. +name: "env-extcred-app" +credentialsId: "" +isServerSideMerge: false +labels: + - "Instance-env-extcred" +cleanInstallApprovalRequired: false +mergeDeployParametersAndE2EParameters: false +deployParameters: + APP_NAMESPACE: "env-extcred-app" + DB_ADMIN_CHECK: + CONFIG_DEPLOY: + param_type: "deploy" + DB_ADMIN_SECRET: + $type: "credRef" + credId: "app-sidecar-token" + SECURED_FLAG: "true" + ENVGENE_CONFIG_REF_NAME: "branch_name" + ENVGENE_CONFIG_TAG: "No Ref tag" + SECRET_FLOW: "helm-values" + SHARED_PASSWORD: + $type: "credRef" + credId: "shared-cred" + property: "password" + Variable_1: "" +e2eParameters: + APP_NAMESPACE: "env-extcred-app" + Variable_2: "" +technicalConfigurationParameters: + APP_NAMESPACE: "env-extcred-app" + Variable_3: "" +deployParameterSets: [] +e2eParameterSets: [] +technicalConfigurationParameterSets: [] diff --git a/test_data/test_environments/cluster-01/env-extcred/cloud.yml b/test_data/test_environments/cluster-01/env-extcred/cloud.yml new file mode 100644 index 000000000..e72abdaed --- /dev/null +++ b/test_data/test_environments/cluster-01/env-extcred/cloud.yml @@ -0,0 +1,79 @@ +# The contents of this file is generated from template artifact: deployment-configuration-env-templates. +# Contents will be overwritten by next generation. +# Please modify this contents only for development purposes or as workaround. +name: "cluster_01_env_extcred" +apiUrl: "api.cluster-01.qubership.org" # cloud passport: ext-mode version: 1.5 +apiPort: "6443" # cloud passport: ext-mode version: 1.5 +privateUrl: "cluster-01.qubership.org" # cloud passport: ext-mode version: 1.5 +publicUrl: "cluster-01.qubership.org" # cloud passport: ext-mode version: 1.5 +dashboardUrl: "https://dashboard.cluster-01.qubership.org" # cloud passport: ext-mode version: 1.5 +labels: [] +defaultCredentialsId: "app-sidecar-token" # cloud passport: ext-mode version: 1.5 +protocol: "https" # cloud passport: ext-mode version: 1.5 +dbMode: "db" +databases: [] +maasConfig: + credentialsId: "app-maas-cred" # cloud passport: ext-mode version: 1.5 + enable: true # cloud passport: ext-mode version: 1.5 + maasUrl: "http://maas.cluster-01.qubership.org" # cloud passport: ext-mode version: 1.5 + maasInternalAddress: "http://maas.maas:8080" # cloud passport: ext-mode version: 1.5 +vaultConfig: + credentialsId: "" + enable: false + url: "" +dbaasConfigs: + - credentialsId: "app-dbaas-cred" # cloud passport: ext-mode version: 1.5 + enable: true # cloud passport: ext-mode version: 1.5 + apiUrl: "http://dbaas.dbaas:8080" # cloud passport: ext-mode version: 1.5 + aggregatorUrl: "https://dbaas.cluster-01.qubership.org" # cloud passport: ext-mode version: 1.5 +consulConfig: + tokenSecret: "app-sidecar-token" # cloud passport: ext-mode version: 1.5 + enabled: true # cloud passport: ext-mode version: 1.5 + publicUrl: "http://consul.consul:8080" # cloud passport: ext-mode version: 1.5 + internalUrl: "http://consul.consul:8080" # cloud passport: ext-mode version: 1.5 +deployParameters: + CLOUD_DASHBOARD_URL: "https://dashboard.cluster-01.qubership.org" # cloud passport: ext-mode version: 1.5 + CMDB_URL: "https://cluster-01.qubership.org" # cloud passport: ext-mode version: 1.5 + CONSUL_ENABLED: true # cloud passport: ext-mode version: 1.5 + DB_ADMIN_USERNAME: + $type: "credRef" + credId: "app-db-cred" + property: "username" + DEFAULT_TENANT_ADMIN_LOGIN: "admin" # cloud passport: ext-mode version: 1.5 + DEFAULT_TENANT_ADMIN_PASSWORD: "password" # cloud passport: ext-mode version: 1.5 + DEFAULT_TENANT_NAME: "tenant" # cloud passport: ext-mode version: 1.5 + DOC_STORAGE_PERSISTENT_BUCKET_NAME: "permanent-bucket" # cloud passport: ext-mode version: 1.5 + DOC_STORAGE_TEMPORARY_BUCKET_NAME: "temporary-bucket" # cloud passport: ext-mode version: 1.5 + GRAFANA_UI_URL: "https://cluster-01.qubership.org" # cloud passport: ext-mode version: 1.5 + GRAYLOG_UI_URL: "https://cluster-01.qubership.org" # cloud passport: ext-mode version: 1.5 + MAAS_PASSWORD: + $type: "credRef" + credId: "app-maas-cred" + property: "password" + MAVEN_REPO_DEV_NAME: "${MAVEN_REPO_NAME}" # cloud passport: ext-mode version: 1.5 + MAVEN_REPO_NAME: "mvn.group" # cloud passport: ext-mode version: 1.5 + MAVEN_REPO_STAGING_NAME: "${MAVEN_REPO_NAME}" # cloud passport: ext-mode version: 1.5 + MAVEN_REPO_URL: "https://artifactory.qubership.org" # cloud passport: ext-mode version: 1.5 + MONITORING_ENABLED: "true" # cloud passport: ext-mode version: 1.5 + STORAGE_PROVIDER: "s3" # cloud passport: ext-mode version: 1.5 + STORAGE_REGION: "eu-west-1" # cloud passport: ext-mode version: 1.5 + STORAGE_RWO_CLASS: "standard" # cloud passport: ext-mode version: 1.5 + STORAGE_RWX_CLASS: "" # cloud passport: ext-mode version: 1.5 + STORAGE_SERVER_URL: "https://minio.cluster-01.qubership.org" # cloud passport: ext-mode version: 1.5 + TEST_PASSWORD: + $type: "credRef" + credId: "app-test-cred" + property: "password" + TRACING_ENABLED: "false" # cloud passport: ext-mode version: 1.5 + TRACING_HOST: "tracing-agent" # cloud passport: ext-mode version: 1.5 + TRACING_UI_URL: "https://cluster-01.qubership.org" # cloud passport: ext-mode version: 1.5 + ZOOKEEPER_ADDRESS: "zookeeper.zookeeper:2181" # cloud passport: ext-mode version: 1.5 + ZOOKEEPER_URL: "${ZOOKEEPER_ADDRESS}" # cloud passport: ext-mode version: 1.5 +e2eParameters: + CMDB_NAME: "" + TEMPLATE_NAME: "extcred-template" +technicalConfigurationParameters: {} +deployParameterSets: [] +e2eParameterSets: [] +technicalConfigurationParameterSets: [] +productionMode: false # cloud passport: ext-mode version: 1.5 diff --git a/test_data/test_environments/cluster-01/env-extcred/tenant.yml b/test_data/test_environments/cluster-01/env-extcred/tenant.yml new file mode 100644 index 000000000..35869c4e3 --- /dev/null +++ b/test_data/test_environments/cluster-01/env-extcred/tenant.yml @@ -0,0 +1,17 @@ +# The contents of this file is generated from template artifact: deployment-configuration-env-templates. +# Contents will be overwritten by next generation. +# Please modify this contents only for development purposes or as workaround. +name: "tenant" +registryName: "default" +description: "External Cred Sample" +owners: "Qubership team" +gitRepository: "" +defaultBranch: "" +credential: "" +labels: [] +globalE2EParameters: + pipelineDefaultRecipients: "" + recipientsStrategy: "merge" + mergeTenantsAndE2EParameters: false + environmentParameters: {} +deployParameters: {} diff --git a/test_data/test_templates/env_templates/extcred-template.yaml b/test_data/test_templates/env_templates/extcred-template.yaml new file mode 100644 index 000000000..6e4784e21 --- /dev/null +++ b/test_data/test_templates/env_templates/extcred-template.yaml @@ -0,0 +1,8 @@ +--- +tenant: "{{ templates_dir }}/env_templates/extcred-template/tenant.yml.j2" +cloud: + template_path: "{{ templates_dir }}/env_templates/extcred-template/cloud.yml.j2" +external_credential_template: "{{ templates_dir }}/env_templates/extcred-template/external-credentials.yml.j2" +namespaces: + - template_path: "{{ templates_dir }}/env_templates/extcred-template/Namespaces/app.yml.j2" + deploy_postfix: "app" diff --git a/test_data/test_templates/env_templates/extcred-template/Namespaces/app.yml.j2 b/test_data/test_templates/env_templates/extcred-template/Namespaces/app.yml.j2 new file mode 100644 index 000000000..3c0f7f74c --- /dev/null +++ b/test_data/test_templates/env_templates/extcred-template/Namespaces/app.yml.j2 @@ -0,0 +1,44 @@ +--- +name: "{{current_env.name}}-app" +labels: +- "Instance-{{current_env.name}}" +credentialsId: "" +isServerSideMerge: false +cleanInstallApprovalRequired: false +mergeDeployParametersAndE2EParameters: false +deployParameters: + DB_ADMIN_CHECK: + CONFIG_DEPLOY: + param_type: "deploy" + DB_ADMIN_SECRET: + $type: "credRef" + credId: "app-sidecar-token" + SECURED_FLAG: "true" + SHARED_PASSWORD: + $type: "credRef" + credId: "shared-cred" + property: "password" + SECRET_FLOW: "helm-values" + ENVGENE_CONFIG_REF_NAME: "{{ lookup('ansible.builtin.env', 'CI_COMMIT_REF_NAME')| default('No Ref Name') }}" + ENVGENE_CONFIG_TAG: "{{ lookup('ansible.builtin.env', 'CI_COMMIT_TAG')| default('No Ref tag') }}" + APP_NAMESPACE: "{{current_env.name}}-app" + Variable_1: "{{env_definition.envTemplate.additionalTemplateVariables.Variable_1}}" +{% if "Test-App-Name" in current_env.solution_structure %} + TEST_SD_VALUE_1: "TEST_SD_VALUE_1" +{% endif %} +e2eParameters: + APP_NAMESPACE: "{{current_env.name}}-app" + Variable_2: "{{env_definition.envTemplate.additionalTemplateVariables.Variable_2}}" +{% if "Test-App-Name" in current_env.solution_structure %} + TEST_SD_VALUE_2: "TEST_SD_VALUE_2" +{% endif %} +technicalConfigurationParameters: + APP_NAMESPACE: "{{current_env.name}}-app" + Variable_3: "{{env_definition.envTemplate.additionalTemplateVariables.Variable_3}}" +{% if "Test-App-Name" in current_env.solution_structure %} + TEST_SD_VALUE_3: "TEST_SD_VALUE_3" +{% endif %} +deployParameterSets: [] +e2eParameterSets: [] +technicalConfigurationParameterSets: [] + diff --git a/test_data/test_templates/env_templates/extcred-template/cloud.yml.j2 b/test_data/test_templates/env_templates/extcred-template/cloud.yml.j2 new file mode 100644 index 000000000..ae1c94691 --- /dev/null +++ b/test_data/test_templates/env_templates/extcred-template/cloud.yml.j2 @@ -0,0 +1,51 @@ +--- +name: "{{current_env.cloud}}" +apiUrl: "{{current_env.cluster.cloud_api_url}}" +apiPort: "{{current_env.cluster.cloud_api_port}}" +privateUrl: "" +publicUrl: "{{current_env.cluster.cloud_public_url}}" +dashboardUrl: "https://dashboard.{{current_env.cluster.cloud_public_url}}" +labels: [] +protocol: "{{current_env.cluster.cloud_api_protocol}}" +dbMode: "db" +databases: [] +defaultCredentialsId: "" +maasConfig: + credentialsId: "maas" + maasUrl: "http://maas.{{current_env.cluster.cloud_public_url}}" + maasInternalAddress: "http://maas.maas:8888" + enable: true +vaultConfig: + url: "" + credentialsId: "" + enable: false +dbaasConfigs: + - credentialsId: "dbaas" + apiUrl: 'http://dbaas.dbaas:8888' + aggregatorUrl: 'https://dbaas.{{current_env.cluster.cloud_public_url}}' + enable: true +consulConfig: + tokenSecret: "consul-token" + publicUrl: 'https://consul.{{current_env.cluster.cloud_public_url}}' + enabled: true + internalUrl: 'http://consul.consul:8888' +e2eParameters: + CMDB_NAME: "{{current_env.cmdb_name}}" + TEMPLATE_NAME: "{{current_env.env_template}}" +deployParameters: + DB_ADMIN_USERNAME: + $type: "credRef" + credId: "app-db-cred" + property: "username" + MAAS_PASSWORD: + $type: "credRef" + credId: "app-maas-cred" + property: "password" + TEST_PASSWORD: + $type: "credRef" + credId: "app-test-cred" + property: "password" +technicalConfigurationParameters: {} +deployParameterSets: [] +e2eParameterSets: [] +technicalConfigurationParameterSets: [] diff --git a/test_data/test_templates/env_templates/extcred-template/external-credentials.yml.j2 b/test_data/test_templates/env_templates/extcred-template/external-credentials.yml.j2 new file mode 100644 index 000000000..5d268e260 --- /dev/null +++ b/test_data/test_templates/env_templates/extcred-template/external-credentials.yml.j2 @@ -0,0 +1,17 @@ +--- +# Renders into Environment Credentials File; must match credential.schema.json per entry. +app-db-cred: + type: external + create: false + secretStore: azure-store + properties: + - name: username + - name: password +app-test-cred: + type: external + create: true + secretStore: gcp-store + properties: + - name: username + - name: password + remoteRefPath: "{{ current_env.cloud }}/{{ current_env.name }}/envgene" diff --git a/test_data/test_templates/env_templates/extcred-template/tenant.yml.j2 b/test_data/test_templates/env_templates/extcred-template/tenant.yml.j2 new file mode 100644 index 000000000..96f19549c --- /dev/null +++ b/test_data/test_templates/env_templates/extcred-template/tenant.yml.j2 @@ -0,0 +1,14 @@ +name: "{{_tenant}}" +registryName: "default" +description: "{{current_env.description}}" +owners: "{{current_env.owners}}" +deployParameters: {} +globalE2EParameters: + pipelineDefaultRecipients: "" + recipientsStrategy: "merge" + mergeTenantsAndE2EParameters: false + environmentParameters: {} +gitRepository: "" +defaultBranch: "" +credential: "" +labels: [] diff --git a/test_data/test_templates/parameters/extcred.yaml b/test_data/test_templates/parameters/extcred.yaml new file mode 100644 index 000000000..93e706417 --- /dev/null +++ b/test_data/test_templates/parameters/extcred.yaml @@ -0,0 +1,12 @@ +version: 1 +name: extcred +parameters: {} +applications: + - appName: "eso-app" + parameters: + TEST_NAMESPACE: "apps-test" + SECRET_FLOW: external-values + - appName: "vals-app" + parameters: + INTEGRATED_SYSTEMS: ABC + TEST_NAMESPACE: "apps-test" \ No newline at end of file