feat: Changes to support external credentials in envgene

python/envgene/envgenehelper/creds_helper.py

@@ -1,8 +1,10 @@
 import re
+import os
 from pathlib import Path
 
 from envgenehelper import crypt, getenv_with_error, get_env_instances_dir, findAllYamlsInDir, openYaml, getEnvCredentialsPath
 from envgenehelper.errors import ValidationError
+from envgenehelper.yaml_helper import  store_value_to_yaml, writeYamlToFile, beautifyYaml, yaml
 
 from .logger import logger
 
@@ -251,7 +253,7 @@ def validate_creds(creds_path: str = ""):
 def check_cred_value(credId, credValue) -> str:
     result = ""
     type = credValue["type"]
-    data = credValue["data"]
+    data = credValue.get("data", {})
     match type:
         case _ if type == CRED_TYPE_USERPASS:
             if is_envgenenullvalue(data["username"]) or is_envgenenullvalue(data["password"]):
@@ -273,3 +275,44 @@ def is_envgenenullvalue(value: object) -> bool:
     if value.lower() == "envgenenullvalue":
         return True
     return False
+
+def extract_external_cred(cred_map):
+    if cred_map.get("$type") != "credRef":
+        return None
+    cred_id = cred_map.get("credId")
+    if not cred_id or not str(cred_id).strip():
+        raise ValueError(f"Invalid credRef: 'credId' is missing or empty in {cred_map}")
+    return cred_id
+
+def validate_cred_types(credsMap, isExternalCredEnv, credFile):
+    types = {
+        v.get("type")
+        for v in credsMap.values()
+        if isinstance(v, dict) and v.get("type")
+    }
+    if not types:
+        return
+    if isExternalCredEnv:
+        if types != {"external"}:
+            raise ValueError(f"Only external credentials allowed. Found: {types} in {credFile}")
+    else:
+        if "external" in types:
+            raise ValueError(f"External credentials not allowed. Found: {types} in {credFile}")
+
+def has_external_creds(credsMap):
+    return any(
+        isinstance(v, dict) and v.get("type") == "external"
+        for v in credsMap.values()
+    )
+
+def copy_creds_to_env_creds_file(env_dir, credsYamlContent, comment, credsSchema):
+    envCredentialsPath = f"{env_dir}/Credentials/credentials.yml"       
+    if os.path.exists(envCredentialsPath) :
+        envCredsYaml = openYaml(envCredentialsPath)
+    else:
+        envCredsYaml = yaml.load("{}")
+    for key, value in credsYamlContent.items() :
+        store_value_to_yaml(envCredsYaml, key, value, comment)
+    # storing credentials yaml
+    writeYamlToFile(envCredentialsPath, envCredsYaml)
+    beautifyYaml(envCredentialsPath, credsSchema)

python/envgene/envgenehelper/yaml_helper.py

@@ -141,10 +141,7 @@ def alignYamlComments(yamlContent, extra_indent=0):
 
 
 def sortYaml(yaml_data, schema_path, remove_additional_props):
-    with open(schema_path, 'r') as f:
-        schema_data = json.load(f)
-    logger.debug(f'Checking yaml with schema: {schema_path}')
-    jsonschema.validate(yaml_data, schema_data)
+    schema_data = validateSchema(yaml_data, schema_path)
     sort_data = jschon_tools.process_json_doc(
         schema_data=schema_data,
         doc_data=yaml_data,
@@ -153,6 +150,13 @@ def sortYaml(yaml_data, schema_path, remove_additional_props):
     )
     return sort_data
 
+def validateSchema(yaml_data, schema_path):
+    with open(schema_path, 'r') as f:
+        schema_data = json.load(f)
+    logger.debug(f'Checking yaml with schema: {schema_path}')
+    jsonschema.validate(yaml_data, schema_data)
+    return schema_data
+
 
 def get_nested_yaml_attribute_or_fail(yaml_content, attribute_str):
     keys = attribute_str.split('.')

schemas/application.sbom.schema.json

@@ -992,4 +992,4 @@
       }
     }
   }
-}
+}

schemas/credential.schema.json

@@ -168,4 +168,4 @@
       }
     ]
   }
-}
+}

scripts/build_env/build_env.py

@@ -466,7 +466,7 @@ def process_additional_template_parameters(render_env_dir, source_env_dir, all_i
 
 
 def build_env(env_name, env_instances_dir, parameters_dir, env_template_dir, resource_profiles_dir,
-              env_specific_resource_profile_map, all_instances_dir, render_context, templates_dirs=None):
+              env_specific_resource_profile_map, all_instances_dir, render_context, templates_dirs=None, isExternalCredEnv=False):
     # Check which role-specific templates were downloaded
     templates_dirs = templates_dirs or {}
     origin_template_exists = NamespaceRole.ORIGIN in templates_dirs
@@ -525,7 +525,7 @@ def build_env(env_name, env_instances_dir, parameters_dir, env_template_dir, res
         header_text=generated_header_text,
         process_env_specific=False)
     # process cloud passport
-    process_cloud_passport(env_dir, env_instances_dir, all_instances_dir)
+    process_cloud_passport(env_dir, env_instances_dir, all_instances_dir, isExternalCredEnv)
     logger.info("Processing cloud with env specific parameters.")
     processTemplate(
         cloudTemlatePath,

scripts/build_env/cloud_passport.py

@@ -20,13 +20,15 @@ def process_and_update_key(targetKey, targetYaml, sourceKey, sourceYaml, comment
     merge_dict_key_with_comment(targetKey, targetYaml, sourceKey, sourceYaml, comment)
     del sourceYaml[sourceKey]
 
-def mergeDeployParametersFromPassport(cloudPassportYaml, cloudYaml, comment) :
+def mergeDeployParametersFromPassport(cloudPassportYaml, cloudYaml, comment, isExternalCredEnv) :
     for domain in cloudPassportYaml:
         if domain == "version": continue
         for paramKey, paramValue in cloudPassportYaml[domain].items():
+            if isExternalCredEnv and paramKey == "SECRET_FLOW" and paramKey in cloudYaml["deployParameters"]:
+                continue
             store_value_to_yaml(cloudYaml["deployParameters"], paramKey, paramValue, comment)
 
-def process_cloud_definition(cloudPassportYaml, env_dir, comment) :
+def process_cloud_definition(cloudPassportYaml, env_dir, comment, isExternalCredEnv) :
     cloud_schema="schemas/cloud.schema.json"
     # cloud
     cloudYamlPath = f"{env_dir}/cloud.yml"
@@ -41,7 +43,7 @@ def process_cloud_definition(cloudPassportYaml, env_dir, comment) :
         maasPassportYaml = cloudPassportYaml["maas"]
         # if maas is presented in cloud passport - than maas is enabled
         store_value_to_yaml(cloudYaml["maasConfig"], "enable", True, comment)
-        store_value_to_yaml(cloudYaml["maasConfig"], "credentialsId", get_cred_id_from_cred_macros(maasPassportYaml["MAAS_CREDENTIALS_USERNAME"]), comment)
+        store_value_to_yaml(cloudYaml["maasConfig"], "credentialsId", extract_cred_id(maasPassportYaml["MAAS_CREDENTIALS_USERNAME"], isExternalCredEnv), comment)
         process_and_update_key("maasUrl", cloudYaml["maasConfig"], "MAAS_SERVICE_ADDRESS", maasPassportYaml, comment)
         process_and_update_key("maasInternalAddress", cloudYaml["maasConfig"], "MAAS_INTERNAL_ADDRESS", maasPassportYaml, comment)
         del cloudPassportYaml["maas"]
@@ -51,7 +53,7 @@ def process_cloud_definition(cloudPassportYaml, env_dir, comment) :
         vaultPassportYaml = cloudPassportYaml["vault"]
         vaultUrl = vaultPassportYaml["VAULT_ADDR"]
         if vaultUrl :
-            store_value_to_yaml(cloudYaml["vaultConfig"], "credentialsId", get_cred_id_from_cred_macros(vaultPassportYaml["VAULT_AUTH_ROLE_ID"]), comment)
+            store_value_to_yaml(cloudYaml["vaultConfig"], "credentialsId", extract_cred_id(vaultPassportYaml["VAULT_AUTH_ROLE_ID"], isExternalCredEnv), comment)
             store_value_to_yaml(cloudYaml["vaultConfig"], "enable", True, comment)
             store_value_to_yaml(cloudYaml["vaultConfig"], "url", vaultUrl, comment)
         else :
@@ -67,7 +69,7 @@ def process_cloud_definition(cloudPassportYaml, env_dir, comment) :
             cloudYaml["dbaasConfigs"] = yaml.load("[]")
             cloudYaml["dbaasConfigs"].append(yaml.load("enable: false"))
         dbaasConfigYaml = cloudYaml["dbaasConfigs"][0]
-        store_value_to_yaml(dbaasConfigYaml, "credentialsId", get_cred_id_from_cred_macros(dbaasPassportYaml["DBAAS_CLUSTER_DBA_CREDENTIALS_USERNAME"]), comment)
+        store_value_to_yaml(dbaasConfigYaml, "credentialsId", extract_cred_id(dbaasPassportYaml["DBAAS_CLUSTER_DBA_CREDENTIALS_USERNAME"], isExternalCredEnv), comment)
         store_value_to_yaml(dbaasConfigYaml, "enable", True, comment)
         process_and_update_key("apiUrl", dbaasConfigYaml, "API_DBAAS_ADDRESS", dbaasPassportYaml, comment)
         process_and_update_key("aggregatorUrl", dbaasConfigYaml, "DBAAS_AGGREGATOR_ADDRESS", dbaasPassportYaml, comment)
@@ -78,7 +80,7 @@ def process_cloud_definition(cloudPassportYaml, env_dir, comment) :
     if "consul" in cloudPassportYaml :
           consulPassportYaml = cloudPassportYaml["consul"]
           consulConfigYaml = cloudYaml["consulConfig"]
-          store_value_to_yaml(consulConfigYaml, "tokenSecret", get_cred_id_from_cred_macros(consulPassportYaml["CONSUL_ADMIN_TOKEN"]), comment)
+          store_value_to_yaml(consulConfigYaml, "tokenSecret", extract_cred_id(consulPassportYaml["CONSUL_ADMIN_TOKEN"], isExternalCredEnv), comment)
           process_and_update_key("enabled", consulConfigYaml, "CONSUL_ENABLED", consulPassportYaml, comment)
           process_and_update_key("publicUrl", consulConfigYaml, "CONSUL_PUBLIC_URL", consulPassportYaml, comment)
           process_and_update_key("internalUrl", consulConfigYaml, "CONSUL_URL", consulPassportYaml, comment)
@@ -87,14 +89,15 @@ def process_cloud_definition(cloudPassportYaml, env_dir, comment) :
           del cloudPassportYaml["consul"]
     else:
         store_value_to_yaml(cloudYaml["consulConfig"], "enabled", False)
+
     # adding rest of cloud passport parameters to cloud deploy parameters
     logger.debug(f"Rest of params from cloud passport are: \n{dump_as_yaml_format(cloudPassportYaml)}")
-    mergeDeployParametersFromPassport(cloudPassportYaml, cloudYaml, comment)
+    mergeDeployParametersFromPassport(cloudPassportYaml, cloudYaml, comment, isExternalCredEnv)
     # storing cloud yaml
     writeYamlToFile(cloudYamlPath, cloudYaml)
     beautifyYaml(cloudYamlPath, cloud_schema)
 
-def add_cloud_passport_creds(cloud_passport_name, cloud_passport_file_path, env_dir, comment):
+def add_cloud_passport_creds(cloud_passport_name, cloud_passport_file_path, env_dir, comment, isExternalCredEnv):
     logger.info(f"Searching credentials for cloud passport {cloud_passport_file_path}")
     credsSchema="schemas/credential.schema.json"
     # first searching in subfolder
@@ -111,16 +114,19 @@ def add_cloud_passport_creds(cloud_passport_name, cloud_passport_file_path, env_
     else:
         logger.error(f"No cloud pasport credentials files found in either {passportSubfolderPath} or {passportSameFolderPath}.")
         raise ReferenceError(f"No cloud pasport credentials files found. See logs above")
-    envCredentialsPath = f"{env_dir}/Credentials/credentials.yml"
-    if os.path.exists(envCredentialsPath) :
-        envCredsYaml = openYaml(envCredentialsPath)
-    else:
-        envCredsYaml = yaml.load("{}")
-    for key, value in passportCredsYaml.items() :
-        store_value_to_yaml(envCredsYaml, key, value, comment)
-    # storing credentials yaml
-    writeYamlToFile(envCredentialsPath, envCredsYaml)
-    beautifyYaml(envCredentialsPath, credsSchema)
+    copy_creds_to_env_creds_file(env_dir, passportCredsYaml, comment, credsSchema)  
+
+def extract_cred_id(param, isExternal=False):
+    if isExternal:
+        if isinstance(param, dict):
+            cred_id = extract_external_cred(param)
+            if cred_id is None:
+                raise ValueError(f"Invalid external cred: expected '$type'='credRef' with valid 'credId' in {param} in cloud passport")
+            return cred_id
+        else:
+            return get_cred_id_from_cred_macros(param)
+    return get_cred_id_from_cred_macros(param)
+
 
 def update_env_definition_with_cloud_name(render_env_dir, source_env_dir, all_instances_dir):
     inventoryYaml = getEnvDefinition(render_env_dir)
@@ -132,7 +138,7 @@ def update_env_definition_with_cloud_name(render_env_dir, source_env_dir, all_in
     else:
         logger.info(f"No cloud name found for env {render_env_dir} from passport")
 
-def process_cloud_passport(render_env_dir, env_instances_dir, instances_dir) :
+def process_cloud_passport(render_env_dir, env_instances_dir, instances_dir, isExternalCredEnv=False) :
     logger.info(f"Trying to find cloud passport definition file")
     cloudPassportFilePath = find_cloud_passport_definition(env_instances_dir, instances_dir)
     cloudPassportFileName = extractNameFromFile(cloudPassportFilePath)
@@ -142,7 +148,7 @@ def process_cloud_passport(render_env_dir, env_instances_dir, instances_dir) :
         logger.info(f"Processing cloud passport: {cloudPassportFilePath}")
         cloudPassportYaml = openYaml(cloudPassportFilePath)
         comment = f"cloud passport: {cloudPassportFileName} version: {cloudPassportYaml['version']}"
-        process_cloud_definition(cloudPassportYaml, render_env_dir, comment)
-        add_cloud_passport_creds(cloudPassportFileName, cloudPassportFilePath, render_env_dir, comment)
+        process_cloud_definition(cloudPassportYaml, render_env_dir, comment, isExternalCredEnv)
+        add_cloud_passport_creds(cloudPassportFileName, cloudPassportFilePath, render_env_dir, comment, isExternalCredEnv)
     else:
         logger.info("No cloud passport definition found. Cloud passport processing skipped...")