diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/FileSystemUtils.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/FileSystemUtils.java index 61d70e14c..399adb796 100644 --- a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/FileSystemUtils.java +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/FileSystemUtils.java @@ -68,9 +68,6 @@ public void createEffectiveSetFolder(Optional solutionDescriptor private void createEffectiveSetTwo(Optional solutionDescriptor) throws IOException { File file = getFileFromGivenPath(data.getOutputDir()); - if (file.exists()) { - FileUtils.forceDelete(file); - } file.mkdir(); Path pipelinePath = getFileFromGivenPath(data.getOutputDir(), "pipeline").toPath(); Files.createDirectories(pipelinePath); @@ -92,13 +89,9 @@ private void createEffectiveSetTwo(Optional solutionDescriptor) })); } - private void createEffectiveSetOne(List applicationDTOList) throws IOException { + private void createEffectiveSetOne(List applicationDTOList) { File file = getFileFromGivenPath(data.getOutputDir()); - if (file.exists()) { - FileUtils.forceDelete(file); - } file.mkdir(); - applicationDTOList .forEach(app -> { Path appPath = getFileFromGivenPath(data.getOutputDir(), app.getNamespace(), app.getAppName()).toPath(); diff --git a/build_effective_set_generator/scripts/main.py b/build_effective_set_generator/scripts/crypt_manager.py similarity index 100% rename from build_effective_set_generator/scripts/main.py rename to build_effective_set_generator/scripts/crypt_manager.py diff --git a/build_effective_set_generator/scripts/effective_set_entrypoint.py b/build_effective_set_generator/scripts/effective_set_entrypoint.py new file mode 100644 index 000000000..3279677cf --- /dev/null +++ b/build_effective_set_generator/scripts/effective_set_entrypoint.py @@ -0,0 +1,164 @@ +import subprocess +from os import getenv + +from envgenehelper.business_helper import get_environment_name_from_full_name, get_current_env_dir_from_env_vars +from envgenehelper.file_helper import delete_dir, deleteFileIfExists, delete_dir_if_exists +from envgenehelper.json_helper import openJson +from envgenehelper.logger import logger +from envgenehelper.yaml_helper import writeYamlToFile, openYaml +from envgenehelper.effective_set_helper import resolve_es_generation_mode, GenerationMode, PartialMergeMode, \ + resolve_partial_merge_mode, ES_MAPPING_FILE, ESGenerationContext +from envgenehelper.sd_helper import get_sd_dir, DELTA_SD_FILE_NAME, SD_FILE_NAME + +from handle_effective_set_config import handle_effective_set_config + + +def effective_set_entrypoint(): + full_env_name = getenv("FULL_ENV_NAME") + effective_set_dir = get_current_env_dir_from_env_vars() / "effective-set" + sd_path = get_sd_dir().joinpath(SD_FILE_NAME) + delta_sd_path = get_sd_dir().joinpath(DELTA_SD_FILE_NAME) + + if resolve_es_generation_mode() == GenerationMode.PARTIAL: + if not delta_sd_path.exists() and sd_path.exists(): + raise ValueError( + f"[Partial effective set generation] impossible without delta sd {delta_sd_path} and full sd {sd_path}") + # first run + if not effective_set_dir.exists(): + _run_full_generation(effective_set_dir, full_env_name, sd_path) + + elif resolve_partial_merge_mode() == PartialMergeMode.REVERSE: + _run_reverse_merge(effective_set_dir, full_env_name, delta_sd_path) + else: + _run_forward_merge(effective_set_dir, full_env_name, delta_sd_path) + else: + _run_full_generation(effective_set_dir, full_env_name, sd_path) + + # do not commit delta sd to repo + deleteFileIfExists(delta_sd_path) + + +def _run_full_generation(effective_set_dir, full_env_name, sd_path): + cmd = _build_cli_cmd(effective_set_dir, full_env_name, sd_path) + delete_dir(effective_set_dir) + subprocess.run(["sh", cmd], check=True) + + +def _run_forward_merge(effective_set_dir, full_env_name, delta_sd_path): + cmd = _build_cli_cmd(effective_set_dir, full_env_name, delta_sd_path) + + delta_sd = openYaml(delta_sd_path) + apps = delta_sd.get("applications", []) + + deploy_postfixes = { + app.get("deployPostfix") + for app in apps + if app.get("deployPostfix") + } + + for ns in deploy_postfixes: + delete_dir_if_exists(effective_set_dir / ESGenerationContext.CLEANUP.value / ns) + + for app in apps: + app_name = app.get("version", "").split(":")[0] + ns = app.get("deployPostfix") + delete_dir_if_exists(effective_set_dir / ESGenerationContext.RUNTIME.value / ns / app_name) + delete_dir_if_exists(effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ns / app_name) + + delete_dir_if_exists(effective_set_dir / ESGenerationContext.TOPOLOGY.value) + delete_dir_if_exists(effective_set_dir / ESGenerationContext.PIPELINE.value) + + cleanup_mapping_path = effective_set_dir / ESGenerationContext.CLEANUP.value / ES_MAPPING_FILE + runtime_mapping_path = effective_set_dir / ESGenerationContext.RUNTIME.value / ES_MAPPING_FILE + deployment_mapping_path = effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ES_MAPPING_FILE + + cleanup_mapping = openYaml(cleanup_mapping_path, allow_default=True) + runtime_mapping = openYaml(runtime_mapping_path, allow_default=True) + deployment_mapping = openYaml(deployment_mapping_path, allow_default=True) + + subprocess.run(["sh", cmd], check=True) + + new_cleanup_mapping = openYaml(cleanup_mapping_path, allow_default=True) + new_runtime_mapping = openYaml(runtime_mapping_path, allow_default=True) + new_deployment_mapping = openYaml(deployment_mapping_path, allow_default=True) + + cleanup_mapping.update(new_cleanup_mapping) + runtime_mapping.update(new_runtime_mapping) + deployment_mapping.update(new_deployment_mapping) + + writeYamlToFile(cleanup_mapping_path, cleanup_mapping) + writeYamlToFile(runtime_mapping_path, runtime_mapping) + writeYamlToFile(deployment_mapping_path, deployment_mapping) + + +def _run_reverse_merge(effective_set_dir, full_env_name, delta_sd_path): + apps = openYaml(delta_sd_path).get("applications", []) + + mapping_paths = [ + effective_set_dir / ESGenerationContext.CLEANUP.value / ES_MAPPING_FILE, + effective_set_dir / ESGenerationContext.RUNTIME.value / ES_MAPPING_FILE, + effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ES_MAPPING_FILE, + ] + ns = get_environment_name_from_full_name(full_env_name) + + for app in apps: + app_name = app.get("version", "").split(":")[0] + dp = app.get("deployPostfix") + + runtime_dp = effective_set_dir / ESGenerationContext.RUNTIME.value / dp + deployment_dp = effective_set_dir / ESGenerationContext.DEPLOYMENT.value / dp + cleanup_dp = effective_set_dir / ESGenerationContext.CLEANUP.value / dp + + delete_dir(runtime_dp / app_name) + delete_dir(deployment_dp / app_name) + + if any(runtime_dp.iterdir()): + continue + + delete_dir(runtime_dp) + delete_dir(deployment_dp) + delete_dir(cleanup_dp) + + for path in mapping_paths: + if not path.exists(): + logger.warning(f"Mapping file not found, skipping: {path}") + continue + mapping = openYaml(path, allow_default=True) or {} + if ns not in mapping: + logger.warning(f"Namespace '{ns}' not found in mapping file {path}, skipping removing by key: {ns}") + continue + mapping.pop(ns) + writeYamlToFile(path, mapping) + + +def _build_cli_cmd(effective_set_dir, full_env_name, sd_path): + cmd = [ + "/module/scripts/utils/run_effective_set_cli.sh", + f"--env-id={full_env_name}", + "--envs-path=$CI_PROJECT_DIR/environments", + f"--output={effective_set_dir}", + ] + + if sd_path.is_file(): + cmd.extend([ + "--registries=${CI_PROJECT_DIR}/configuration/registry.yml", + "--sboms-path=$sboms_path", + "--sd-path=$sd_path", + ]) + + effective_set_config = getenv("EFFECTIVE_SET_CONFIG") + if effective_set_config: + handle_effective_set_config(effective_set_config) + effective_set_output = openJson("/tmp/effective_set_output.json") + extra_args = effective_set_output.get("extra_args") or [] + cmd.extend(extra_args) + + deployment_id = getenv("DEPLOYMENT_SESSION_ID") + if deployment_id: + cmd.append(f"--extra_params=DEPLOYMENT_SESSION_ID={deployment_id}") + + custom_params = getenv("CUSTOM_PARAMS") + if custom_params: + cmd.append(f"--custom-params={custom_params}") + + return cmd diff --git a/build_pipegene/scripts/effective_set_job.py b/build_pipegene/scripts/effective_set_job.py index 2db54fecf..88142d15a 100644 --- a/build_pipegene/scripts/effective_set_job.py +++ b/build_pipegene/scripts/effective_set_job.py @@ -1,93 +1,39 @@ import json +import os from os import getenv, environ from pathlib import Path +from envgenehelper import logger, copy_path from gcip import WhenStatement, Need -from envgenehelper import logger, get_sboms_dir -from envgenehelper import cleanup_targets from pipeline_helper import job_instance def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluster_name, params): logger.info(f'Prepare generate-effective-set job for {full_env_name}') - logger.info(f'Cleanup_targets: {cleanup_targets}') - - app_reg_defs_job = params["APP_REG_DEFS_JOB"] - artifact_app_defs_path = params["APP_DEFS_PATH"] - artifact_reg_defs_path = params["REG_DEFS_PATH"] - sd_version = params["SD_VERSION"] - sd_data = params["SD_DATA"] - deployment_id = params["DEPLOYMENT_SESSION_ID"] - effective_set_config = params["EFFECTIVE_SET_CONFIG"] - if "CUSTOM_PARAMS" in params: - custom_params = params["CUSTOM_PARAMS"] - is_local_app_def = artifact_app_defs_path and artifact_reg_defs_path and app_reg_defs_job + effective_set_config = params["EFFECTIVE_SET_CONFIG"] + if effective_set_config: + logger.info(f"EFFECTIVE_SET_CONFIG: {effective_set_config}") + effective_set_config_dict = json.loads(effective_set_config) + validate_topology_context_mode(effective_set_config_dict, full_env_name, params) - base_dir = getenv('CI_PROJECT_DIR') + app_reg_defs_job = params.get("APP_REG_DEFS_JOB") + is_local_app_def = init_local_app_defs_from_artifact(full_env_name, app_reg_defs_job, params) - sd_path = Path(f'{base_dir}/environments/{full_env_name}/Inventory/solution-descriptor/sd.yaml') - # TODO it is necessary to remove unnecessary calls, leave only script calls in such jobs! bad for gsf delivery script = [ - #Overriding sd_path to pick the correct value for CI_PROJECT_DIR - f'base_env_path="$CI_PROJECT_DIR/environments/{full_env_name}";', - 'app_defs_path="$base_env_path/AppDefs";', - 'reg_defs_path="$base_env_path/RegDefs";', - 'sboms_path="$CI_PROJECT_DIR/sboms";', - 'sd_path="$base_env_path/Inventory/solution-descriptor/sd.yaml";', # cert handling for java 'mkdir -p ${CI_PROJECT_DIR}/configuration/certs/', 'if [ -f /default_cert.pem ]; then cp /default_cert.pem "${CI_PROJECT_DIR}/configuration/certs/"; fi', 'for cert in "${CI_PROJECT_DIR}/configuration/certs/*" ; do [ -f "$cert" ] && keytool -import -trustcacerts -alias "$(basename "$cert")" -file "$cert" -keystore /etc/ssl/certs/keystore.jks -storepass changeit -noprompt; done', - 'python3 /module/scripts/main.py decrypt_cred_files', - f'[ -n "$APP_REG_DEFS_JOB" ] && [ -n "$APP_DEFS_PATH" ] && mkdir -p $app_defs_path && cp -rf {artifact_app_defs_path}/* $app_defs_path', - f'[ -n "$APP_REG_DEFS_JOB" ] && [ -n "$REG_DEFS_PATH" ] && mkdir -p $reg_defs_path && cp -fr {artifact_reg_defs_path}/* $reg_defs_path', - 'python3 /module/scripts/main.py validate_creds', - 'python3 /module/scripts/sboms_retention_policy.py' - ] - cmdb_cli_cmd_call = [ - f"/module/scripts/utils/entrypoint.sh --env-id={full_env_name}", - "--envs-path=$CI_PROJECT_DIR/environments", - f"--output=$CI_PROJECT_DIR/environments/{full_env_name}/effective-set" + 'python3 /module/scripts/crypt_manager.py decrypt_cred_files', + 'python3 /module/scripts/crypt_manager.py validate_creds', + 'python3 /module/scripts/sboms_retention_policy.py', + 'python3 /module/scripts/effective_set_entrypoint.py', + 'python3 /module/scripts/crypt_manager.py encrypt_cred_files', ] - effective_set_config_dict = {} - if effective_set_config: - effective_set_config_dict = json.loads(effective_set_config) - - effective_set_version = effective_set_config_dict.get("version") or "v2.0" - full_sd_exists = sd_path.is_file() - sd_data = bool(sd_data) or bool(sd_version) - - if not (full_sd_exists and sd_data) and effective_set_version.lower() == "v1.0": - raise ValueError("Feature generation effective set for pipeline and topology context is not supported for v1.0") - - if full_sd_exists or sd_data: - cmdb_cli_cmd_call.extend([ - "--registries=${CI_PROJECT_DIR}/configuration/registry.yml", - "--sboms-path=$sboms_path", - "--sd-path=$sd_path", - ]) - - logger.info(f'Prepare generate_effective_set job for {full_env_name}.') - if effective_set_config: - logger.info(f"EFFECTIVE_SET_CONFIG: {effective_set_config}") - script.extend([ - f"python3 /module/scripts/handle_effective_set_config.py --effective-set-config '{effective_set_config}'", - 'extra_args=$(jq -r \'.extra_args // [] | join(" ")\' /tmp/effective_set_output.json)', - ]) - cmdb_cli_cmd_call.extend(["$extra_args"]) - if deployment_id: - cmdb_cli_cmd_call.extend([f"--extra_params=DEPLOYMENT_SESSION_ID={deployment_id}"]) - - if custom_params: - logger.info(f"custom_params : {custom_params}") - cmdb_cli_cmd_call.extend([f"--custom-params='{custom_params}'"]) - script.append(" ".join(cmdb_cli_cmd_call)) - script.append('python3 /module/scripts/main.py encrypt_cred_files') - generate_effective_set_params = { "name": f'generate_effective_set.{full_env_name}', "image": '${effective_set_generator_image}', @@ -101,7 +47,7 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste "ENV_NAME": env_name, "INSTANCES_DIR": "${CI_PROJECT_DIR}/environments", "effective_set_generator_image": "$effective_set_generator_image", - "EXCLUDE_CLEANUP_TARGETS": " ".join(cleanup_targets) + "FULL_ENV_NAME": full_env_name, } needs = [] @@ -113,9 +59,11 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste environ['CI_PIPELINE_ID'] = '0000000' needs.append(Need(job=app_reg_defs_job, pipeline=real_ci_pipe_id, artifacts=True)) environ['CI_PIPELINE_ID'] = real_ci_pipe_id + generate_effective_set_job = job_instance(params=generate_effective_set_params, needs=needs, - vars=generate_effective_set_vars) + vars=generate_effective_set_vars) + effective_set_config_dict = {} effective_set_expiry = effective_set_config_dict.get("effective_set_expiry") or "1 hour" logger.info(f"effective set expiry value '{effective_set_expiry}'") generate_effective_set_job.artifacts.expire_in = effective_set_expiry @@ -124,3 +72,29 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste pipeline.add_children(generate_effective_set_job) return generate_effective_set_job + + +def validate_topology_context_mode(effective_set_config_dict, full_env_name, params): + effective_set_version = effective_set_config_dict.get("version") or "v2.0" + sd_input = bool(params["SD_DATA"]) or bool(params["SD_VERSION"]) + full_sd_path = Path( + f'{getenv('CI_PROJECT_DIR')}/environments/{full_env_name}/Inventory/solution-descriptor/sd.yaml') + any_sd = full_sd_path.exists() and sd_input + # effective set generation in version 1.0 does not support no sd mode + if not any_sd and effective_set_version.lower() == "v1.0": + raise ValueError("Feature generation effective set for pipeline and topology context is not supported for v1.0") + + if not any_sd: + logger.info("No-SD Mode: no SD present, only topology and pipeline contexts are generated; " + "deployment, runtime, and cleanup are skipped, SBOMs are not requested") + + +def init_local_app_defs_from_artifact(full_env_name, app_reg_defs_job, params): + work_dir = os.path.join(getenv('CI_PROJECT_DIR'), "environments", full_env_name) + artifact_app_defs_path = params.get("APP_DEFS_PATH") + artifact_reg_defs_path = params.get("REG_DEFS_PATH") + is_local_app_def = artifact_app_defs_path and artifact_reg_defs_path and app_reg_defs_job + if is_local_app_def: + copy_path(artifact_app_defs_path, os.path.join(work_dir, "AppDefs")) + copy_path(artifact_reg_defs_path, os.path.join(work_dir, "RegDefs")) + return is_local_app_def diff --git a/build_pipegene/scripts/process_sd_job.py b/build_pipegene/scripts/process_sd_job.py index 641671f19..758cfe545 100644 --- a/build_pipegene/scripts/process_sd_job.py +++ b/build_pipegene/scripts/process_sd_job.py @@ -1,14 +1,12 @@ -from os import getenv - +from envgenehelper import logger from gcip import WhenStatement -from envgenehelper import logger from pipeline_helper import job_instance def prepare_process_sd(pipeline, full_env, environment_name, cluster_name): logger.info(f'Prepare process_sd job for {full_env}') - + script = [ f'base_env_path="$CI_PROJECT_DIR/environments/{full_env}";', 'app_defs_path="$base_env_path/AppDefs";', @@ -28,12 +26,12 @@ def prepare_process_sd(pipeline, full_env, environment_name, cluster_name): process_sd_set_vars = { "CLUSTER_NAME": cluster_name, "ENVIRONMENT_NAME": environment_name, - "ENV_NAME": environment_name, "INSTANCES_DIR": "${CI_PROJECT_DIR}/environments", + "FULL_ENV_NAME": full_env, } process_sd_job = job_instance(params=process_sd_set_params, vars=process_sd_set_vars) - process_sd_job.artifacts.when = WhenStatement.ALWAYS + process_sd_job.artifacts.when = WhenStatement.ALWAYS pipeline.add_children(process_sd_job) - - return process_sd_job \ No newline at end of file + + return process_sd_job diff --git a/python/envgene/envgenehelper/__init__.py b/python/envgene/envgenehelper/__init__.py index cace6f169..275d33ac5 100644 --- a/python/envgene/envgenehelper/__init__.py +++ b/python/envgene/envgenehelper/__init__.py @@ -7,7 +7,7 @@ from .collections_helper import * from .logger import logger from .creds_helper import * -from .sd_merge_helper import * +from .sd_helper import * from .yaml_validator import checkByWhiteList, checkByBlackList, checkSchemaValidationFailed, getSchemaValidationErrorMessage from .crypt import decrypt_file, encrypt_file, decrypt_all_cred_files_for_env, encrypt_all_cred_files_for_env, is_encrypted -from .constants import cleanup_targets \ No newline at end of file +from .constants import cleanup_targets diff --git a/python/envgene/envgenehelper/business_helper.py b/python/envgene/envgenehelper/business_helper.py index 86879dd10..de01528ac 100644 --- a/python/envgene/envgenehelper/business_helper.py +++ b/python/envgene/envgenehelper/business_helper.py @@ -169,7 +169,7 @@ def getTemplateArtifactName(env_definition_yaml): return gav["artifact_id"] -def getEnvDefinition(env_dir = None): +def getEnvDefinition(env_dir=None): env_dir = env_dir or get_current_env_dir_from_env_vars() env_definition_path = getEnvDefinitionPath(env_dir) if not check_file_exists(env_definition_path): @@ -371,11 +371,13 @@ def find_cloud_name_from_passport(source_env_dir, all_instances_dir): else: return "" + class NamespaceRole(StrEnum): COMMON = auto() ORIGIN = auto() PEER = auto() + def get_namespace_role(ns_name: str, bgd_object: dict | None = None) -> NamespaceRole: if not bgd_object: bgd_object = get_bgd_object() @@ -387,6 +389,7 @@ def get_namespace_role(ns_name: str, bgd_object: dict | None = None) -> Namespac return NamespaceRole.PEER return NamespaceRole.COMMON + @dataclass class NamespaceFile: path: Path @@ -409,6 +412,7 @@ def get_namespaces_path(env_dir: Path | None = None) -> Path: logger.debug(namespaces_path) return namespaces_path + def get_bgd_path(env_dir: Path | None = None) -> Path: env_dir = env_dir or get_current_env_dir_from_env_vars() bgd_path = env_dir.joinpath('bg_domain.yml') @@ -422,6 +426,7 @@ def get_bgd_object(env_dir: Path | None = None) -> CommentedMap: logger.debug(bgd_object) return bgd_object + def get_namespaces(env_dir: Path | None = None) -> list[NamespaceFile]: namespaces_path = get_namespaces_path(env_dir) if not check_dir_exists(str(namespaces_path)): @@ -432,6 +437,7 @@ def get_namespaces(env_dir: Path | None = None) -> list[NamespaceFile]: logger.debug(namespaces) return namespaces + def get_template_dirs(base_dir: str | None = None) -> dict[NamespaceRole, str]: base_dir = base_dir if base_dir else getenv_with_error('CI_PROJECT_DIR') result = {} @@ -444,9 +450,11 @@ def get_template_dirs(base_dir: str | None = None) -> dict[NamespaceRole, str]: result[NamespaceRole.PEER] = peer_template_path return result + def is_from_template_dir(file_path: str) -> bool: return bool(TEMPLATE_DIR_PATTERN.search(file_path)) def get_sboms_dir(work_dir) -> Path: return Path(work_dir) / "sboms" + diff --git a/python/envgene/envgenehelper/constants.py b/python/envgene/envgenehelper/constants.py index f7f2f3c10..a68a33b8d 100644 --- a/python/envgene/envgenehelper/constants.py +++ b/python/envgene/envgenehelper/constants.py @@ -1,4 +1,5 @@ # envgenehelper/constants.py +from enum import Enum cleanup_targets = [ "Applications", diff --git a/python/envgene/envgenehelper/effective_set_helper.py b/python/envgene/envgenehelper/effective_set_helper.py new file mode 100644 index 000000000..63132f2a1 --- /dev/null +++ b/python/envgene/envgenehelper/effective_set_helper.py @@ -0,0 +1,47 @@ +from enum import Enum +from os import getenv + +from envgenehelper import get_envgene_config_yaml, calculate_merge_mode, MergeType + + +class ESGenerationContext(Enum): + TOPOLOGY = "topology" + PIPELINE = "pipeline" + DEPLOYMENT = "deployment" + RUNTIME = "runtime" + CLEANUP = "cleanup" + + +ES_MAPPING_FILE = "mapping.yaml" + + +class GenerationMode(Enum): + FULL = "full" + PARTIAL = "partial" + NO_SD = "no_sd" + + +class PartialMergeMode(Enum): + FORWARD = "forward" + REVERSE = "reverse" + + +def resolve_partial_merge_mode(): + merge_mode = calculate_merge_mode(getenv("SD_REPO_MERGE_MODE"), getenv("SD_DELTA")) + + if merge_mode == MergeType.BASIC_EXCLUSION: + return PartialMergeMode.REVERSE + + if merge_mode in {MergeType.BASIC, MergeType.EXTENDED}: + return PartialMergeMode.FORWARD + + raise ValueError(f"Unsupported merge mode for partial: {merge_mode}") + + +def resolve_es_generation_mode(): + merge_mode = calculate_merge_mode(getenv("SD_REPO_MERGE_MODE"), getenv("SD_DELTA")) + + if get_envgene_config_yaml().get("partial_effective_set_generation") and merge_mode.name != MergeType.REPLACE: + return GenerationMode.PARTIAL + + return GenerationMode.FULL diff --git a/python/envgene/envgenehelper/file_helper.py b/python/envgene/envgenehelper/file_helper.py index 6b547612c..56ca7e27b 100644 --- a/python/envgene/envgenehelper/file_helper.py +++ b/python/envgene/envgenehelper/file_helper.py @@ -1,12 +1,11 @@ -import os import glob +import os import re import shutil import tarfile -import time import zipfile -from typing import Callable from pathlib import Path +from typing import Callable from .logger import logger @@ -303,3 +302,8 @@ def cleanup_dir_by_age(dir_path, keep_last: int): for old_file in files[keep_last:]: logger.info(f"Removing file: {old_file}") deleteFileIfExists(old_file) + + +def delete_dir_if_exists(dir_path): + if os.path.isdir(dir_path): + shutil.rmtree(dir_path) diff --git a/python/envgene/envgenehelper/sd_merge_helper.py b/python/envgene/envgenehelper/sd_helper.py similarity index 75% rename from python/envgene/envgenehelper/sd_merge_helper.py rename to python/envgene/envgenehelper/sd_helper.py index 173498034..999ea4595 100644 --- a/python/envgene/envgenehelper/sd_merge_helper.py +++ b/python/envgene/envgenehelper/sd_helper.py @@ -3,6 +3,8 @@ MERGE_IMPOSSIBLE = "SD merge error:\nDelta SD contains a new applications, but doesn't contain this application in the deployGraph.\nSD Merge is impossible." NEW_CHUNK_ERROR = "SD merge error:\nDelta SD contains a new chunk\nSD Merge is impossible." NO_DEPLOY_GRAPH_ERROR = "SD merge error:\nDelta SD contains deployGraph, but Full SD doesn't contain deployGraph.\nSD Merge is impossible." +SD_FILE_NAME = "sd.yaml" +DELTA_SD_FILE_NAME = "delta_sd.yaml" def get_app_name(name: str): @@ -80,7 +82,8 @@ def add_app(entry, apps: list) -> int: apps.append(entry) return 1 -#TODO : notation is supported only for extended merge for now, but later have to be removed + +# TODO : notation is supported only for extended merge for now, but later have to be removed def extended_merge(full_sd, delta_sd): # Merges delta SD into full SD by updating or adding matching apps, ensuring deployGraph consistency logger.info("Inside extended_merge") @@ -187,10 +190,9 @@ def basic_merge(full_sd, delta_sd): def basic_exclusion_merge(full_sd, delta_sd): """ Merge Delta SD into Full SD using `basic-exclusion-merge` rules: - 1. Matching App => update version from Delta + 1. Matching App, New App => WARN 2. Duplicating App => remove from Full SD - 3. New App => log warning - 4. Output contains only `applications` key + 3. Output contains only `applications` key """ logger.info("Inside basic_exclusion_merge") logger.info(f"Full SD: {full_sd}") @@ -199,7 +201,6 @@ def basic_exclusion_merge(full_sd, delta_sd): delta_apps = delta_sd.get("applications", []) result_apps = [] - # Track matched delta apps matched_delta_indices = set() # Stage 1: Process full SD @@ -207,23 +208,64 @@ def basic_exclusion_merge(full_sd, delta_sd): keep = True for i, d_app in enumerate(delta_apps): if is_duplicating(f_app, d_app): - # Rule 3: Remove duplicating + # Rule 2: Remove duplicating keep = False matched_delta_indices.add(i) break elif is_matching(f_app, d_app): - # Rule 1: Replace matching - result_apps.append(d_app) - keep = False + # Rule 1: Warn about matching apps + logger.warning(f"Warning: Update application '{get_app_name_sd(d_app)}' ignored (matching in Full SD)") matched_delta_indices.add(i) break if keep: result_apps.append(f_app) - # Stage 2: Warn about new apps + # Rule 1: Warn about new apps for i, d_app in enumerate(delta_apps): if i not in matched_delta_indices: # Rule 2: New Application, rejects - logger.info(f"Warning: New application '{get_app_name_sd(d_app)}' ignored (not present in Full SD)") + logger.warning(f"Warning: New application '{get_app_name_sd(d_app)}' ignored (not present in Full SD)") return {"applications": result_apps} + + +class MergeType(Enum): + EXTENDED = "extended-merge" + REPLACE = "replace" + BASIC = "basic-merge" + BASIC_EXCLUSION = "basic-exclusion-merge" + + @classmethod + def from_value(cls, value: str): + if not isinstance(value, str): + raise ValueError(f"SD_REPO_MERGE_MODE value: '{value}' cannot be non-string") + value_lower = value.strip().lower() + for member in cls: + if member.value == value_lower: + return member + valid_values = [member.value for member in cls] + raise ValueError( + f"Invalid SD_REPO_MERGE_MODE: '{value}'. Valid values are: {valid_values}" + ) + + +def calculate_merge_mode(sd_merge_mode, sd_delta) -> MergeType: + if sd_merge_mode is not None: + effective_merge_mode = MergeType.from_value(sd_merge_mode) + # sd_delta var is deprecated + elif sd_delta == "true": + effective_merge_mode = MergeType.EXTENDED + logger.info( + f"SD_REPO_MERGE_MODE not passed. Calculated based on SD_DELTA={sd_delta}: {effective_merge_mode.value}") + elif sd_delta == "false": + effective_merge_mode = MergeType.REPLACE + logger.info( + f"SD_REPO_MERGE_MODE not passed. Calculated based on SD_DELTA={sd_delta}: {effective_merge_mode.value}") + else: + effective_merge_mode = MergeType.BASIC + logger.info(f"SD_REPO_MERGE_MODE not passed. Default value: {effective_merge_mode.value}") + return effective_merge_mode + + +def get_sd_dir() -> Path: + return Path(f'{get_current_env_dir_from_env_vars()}/{INVENTORY_DIR_NAME}/solution-descriptor/') diff --git a/scripts/build_env/process_sd.py b/scripts/build_env/process_sd.py index 47c0d103a..54713ff01 100644 --- a/scripts/build_env/process_sd.py +++ b/scripts/build_env/process_sd.py @@ -1,43 +1,22 @@ import asyncio import json import os -from enum import Enum from os import path, getenv from pathlib import Path -import yaml - import envgenehelper as helper +import yaml from artifact_searcher import artifact from artifact_searcher.utils import models as artifact_models from envgenehelper.business_helper import getenv_and_log, getenv_with_error +from envgenehelper.collections_helper import split_multi_value_param from envgenehelper.env_helper import Environment -from envgenehelper.file_helper import identify_yaml_extension +from envgenehelper.file_helper import identify_yaml_extension, deleteFileIfExists from envgenehelper.logger import logger from envgenehelper.plugin_engine import PluginEngine -from envgenehelper.sd_merge_helper import basic_merge_multiple -from envgenehelper.collections_helper import split_multi_value_param - - -class MergeType(Enum): - EXTENDED = "extended-merge" - REPLACE = "replace" - BASIC = "basic-merge" - BASIC_EXCLUSION = "basic-exclusion-merge" - - @classmethod - def from_value(cls, value: str): - if not isinstance(value, str): - raise ValueError(f"SD_REPO_MERGE_MODE value: '{value}' cannot be non-string") - value_lower = value.strip().lower() - for member in cls: - if member.value == value_lower: - return member - valid_values = [member.value for member in cls] - raise ValueError( - f"Invalid SD_REPO_MERGE_MODE: '{value}'. Valid values are: {valid_values}" - ) - +from envgenehelper.sd_helper import (basic_merge_multiple, MergeType, calculate_merge_mode, + get_sd_dir, SD_FILE_NAME, DELTA_SD_FILE_NAME) +from typing_extensions import deprecated MERGE_METHODS = { MergeType.BASIC: helper.basic_merge, @@ -88,7 +67,7 @@ def handle_deploy_postfix_namespace_transformation(sd_data: dict, namespace_dict def prepare_vars_and_run_sd_handling(): base_dir = getenv_and_log('CI_PROJECT_DIR') - env_name = getenv_and_log('ENV_NAME') + env_name = getenv_and_log('ENVIRONMENT_NAME') cluster = getenv_and_log('CLUSTER_NAME') env = Environment(base_dir, cluster, env_name) @@ -134,30 +113,14 @@ def build_namespace_dict(env) -> dict: def merge_sd(sd_path: Path, sd_data, merge_func): logger.info(f"Final destination! - {sd_path}") full_sd_yaml = helper.openYaml(sd_path) - logger.info(f"full_sd.yaml before merge: {full_sd_yaml}") + logger.info(f"Full sd before merge: {full_sd_yaml}") helper.check_dir_exist_and_create(sd_path.parent) result = merge_func(full_sd_yaml, sd_data) helper.writeYamlToFile(sd_path, result) logger.info(f"Merged data into Target Path! - {result}") -def calculate_merge_mode(sd_merge_mode, sd_delta) -> MergeType: - if sd_merge_mode is not None: - effective_merge_mode = MergeType.from_value(sd_merge_mode) - elif sd_delta == "true": - effective_merge_mode = MergeType.EXTENDED - logger.info( - f"SD_REPO_MERGE_MODE not passed. Calculated based on SD_DELTA={sd_delta}: {effective_merge_mode.value}") - elif sd_delta == "false": - effective_merge_mode = MergeType.REPLACE - logger.info( - f"SD_REPO_MERGE_MODE not passed. Calculated based on SD_DELTA={sd_delta}: {effective_merge_mode.value}") - else: - effective_merge_mode = MergeType.BASIC - logger.info(f"SD_REPO_MERGE_MODE not passed. Default value: {effective_merge_mode.value}") - return effective_merge_mode - - +@deprecated("SD_DELTA is deprecated") def calculate_sd_delta(sd_delta): logger.info(f"printing sd_delta before {sd_delta}") if sd_delta is not None and str(sd_delta).strip() != "": @@ -189,12 +152,13 @@ def multiply_sds_to_single(sds_data, effective_merge_mode): def handle_sd(env, sd_source_type, sd_version, sd_data, sd_delta, sd_merge_mode): - base_sd_path = Path(f'{env.env_path}/Inventory/solution-descriptor/') - + base_sd_path = get_sd_dir() sd_delta = calculate_sd_delta(sd_delta) effective_merge_mode = calculate_merge_mode(sd_merge_mode, sd_delta) helper.check_dir_exist_and_create(base_sd_path) + # do not commit delta sd to repo, delete old ones + deleteFileIfExists(base_sd_path.joinpath(DELTA_SD_FILE_NAME)) if sd_source_type == "artifact": download_sds_with_version(env, base_sd_path, sd_version, effective_merge_mode) elif sd_source_type == "json": @@ -238,13 +202,13 @@ def extract_sds_from_json(env, base_sd_path: Path, sd_data, effective_merge_mode full_sd_from_pipe = multiply_sds_to_single(transformed_data, effective_merge_mode) validate_applications(full_sd_from_pipe, effective_merge_mode) - sd_path = base_sd_path.joinpath("sd.yaml") - sd_delta_path = base_sd_path.joinpath("delta_sd.yaml") + sd_path = base_sd_path.joinpath(SD_FILE_NAME) + sd_delta_path = base_sd_path.joinpath(DELTA_SD_FILE_NAME) if effective_merge_mode == MergeType.REPLACE: logger.info("Inside replace") if helper.check_file_exists(sd_path): full_sd_yaml = helper.openYaml(sd_path) - logger.info(f"full_sd.yaml before replacement: {json.dumps(full_sd_yaml, indent=2)}") + logger.info(f"Full sd before replacement: {json.dumps(full_sd_yaml, indent=2)}") else: logger.info("No existing SD found at destination. Proceeding to write new SD.") helper.check_dir_exist_and_create(path.dirname(sd_path)) @@ -306,7 +270,7 @@ def download_sd_by_appver(app_name: str, version: str, plugins: PluginEngine) -> artifact_info = asyncio.run( artifact.check_artifact_async(app_def, artifact.FileExtension.JSON, version, - auth_headers=auth_headers)) + auth_headers=auth_headers)) if not artifact_info: raise ValueError( f'Solution descriptor content was not received for {app_name}:{version}') diff --git a/scripts/build_env/tests/base_test.py b/scripts/build_env/tests/base_test.py index e170370a5..36200e358 100644 --- a/scripts/build_env/tests/base_test.py +++ b/scripts/build_env/tests/base_test.py @@ -5,8 +5,9 @@ class BaseTest: base_dir = Path(__file__).resolve().parents[3] test_data_dir = base_dir / "test_data" + output_dir = base_dir / "tmp" + ci_project_dir = test_data_dir - expected_dir = test_data_dir os.environ['CI_PROJECT_DIR'] = str(test_data_dir) os.environ['JSON_SCHEMAS_DIR'] = str(base_dir / "schemas") diff --git a/scripts/build_env/tests/env-build/test_paramset_sorting.py b/scripts/build_env/tests/env-build/test_paramset_sorting.py index e88b7d102..cc286d0a2 100644 --- a/scripts/build_env/tests/env-build/test_paramset_sorting.py +++ b/scripts/build_env/tests/env-build/test_paramset_sorting.py @@ -1,4 +1,3 @@ -from envgenehelper.business_helper import is_from_template_dir from build_env import sort_paramsets_with_same_name class TestSortParamsetsWithSameName: diff --git a/scripts/build_env/tests/env_inventory_generation/test_env_inv_generation.py b/scripts/build_env/tests/env_inventory_generation/test_env_inv_generation.py index 871fe92ed..89c5f3b01 100644 --- a/scripts/build_env/tests/env_inventory_generation/test_env_inv_generation.py +++ b/scripts/build_env/tests/env_inventory_generation/test_env_inv_generation.py @@ -3,7 +3,7 @@ import yaml from env_inventory_generation import generate_env_new_approach, Place, resolve_path, INVENTORY, Action -from envgenehelper import get_cluster_name_from_full_name, dumpYamlToStr, get_environment_name_from_full_name, readYaml, \ +from envgenehelper import get_cluster_name_from_full_name, get_environment_name_from_full_name, readYaml, \ is_dir_empty, writeYamlToFile from envgenehelper.test_helpers import TestHelpers from jinja.jinja import create_jinja_env @@ -22,7 +22,7 @@ class TestEnvInvGen(BaseTest): def setup_method(self): self.set_ci_project_dir(FEATURE_TEST_DIR, "output") - self.expected_dir = self.expected_dir.joinpath(FEATURE_TEST_DIR, "expected") + self.expected_dir = self.test_data_dir / FEATURE_TEST_DIR / "expected" TestHelpers.clean_test_dir(self.ci_project_dir) self.env_name = get_environment_name_from_full_name(self.full_env_name) diff --git a/scripts/build_env/tests/sd/test_process_sd_artifact.py b/scripts/build_env/tests/sd/test_process_sd_artifact.py index 071308c21..30ded125c 100644 --- a/scripts/build_env/tests/sd/test_process_sd_artifact.py +++ b/scripts/build_env/tests/sd/test_process_sd_artifact.py @@ -1,9 +1,11 @@ import os +from pathlib import Path +from unittest.mock import patch import pytest -from unittest.mock import patch -from ruamel.yaml import YAML +from envgenehelper.env_helper import Environment +from scripts.build_env.tests.base_test import BaseTest from test_sd_helpers import do_prerequisites, assert_sd_contents, load_test_pipeline_sd_data os.environ['ENVIRONMENT_NAME'] = "temporary" @@ -11,10 +13,8 @@ os.environ['CI_PROJECT_DIR'] = "temporary" from process_sd import handle_sd -from envgenehelper import * -from envgenehelper.env_helper import Environment +from envgenehelper import SD_FILE_NAME, logger, openJson -yaml = YAML() TEST_CASES_POSITIVE = [ "TC-001-098", @@ -32,49 +32,64 @@ "replace": [] } -TEST_SD_DIR = Path(getAbsPath("../../test_data/test_handle_sd")) -OUTPUT_DIR = getAbsPath("../../tmp/test_handle_sd") -SD = "sd.yaml" - - -@pytest.mark.parametrize("test_case_name", TEST_CASES_POSITIVE) -@patch("process_sd.download_sd_by_appver") -def test_sd_positive(mock_download_sd, test_case_name): - env = Environment(str(Path(OUTPUT_DIR, test_case_name)), "cluster-01", "env-01") - do_prerequisites(SD, TEST_SD_DIR, OUTPUT_DIR, test_case_name, env, test_suits_map) - logger.info(f"======TEST HANDLE_SD_ARTIFACT_POSITIVE: {test_case_name}======") - logger.info(f"Starting SD test:" - f"\n\tTest case: {test_case_name}") - - sd_data, sd_source_type, sd_version, sd_delta, sd_merge_mode = load_test_pipeline_sd_data(TEST_SD_DIR, test_case_name) - - file_path = Path(TEST_SD_DIR, test_case_name, f"mock_sd.json") - sd_data = openJson(file_path) - mock_download_sd.return_value = sd_data - - handle_sd(env, sd_source_type, sd_version, sd_data, sd_delta, sd_merge_mode) - actual_dir = os.path.join(env.env_path, "Inventory", "solution-descriptor") - - assert_sd_contents(TEST_SD_DIR, OUTPUT_DIR, test_case_name, actual_dir, test_suits_map) - logger.info(f"=====SUCCESS - {test_case_name}======") - - -@pytest.mark.parametrize("test_case_name,expected_exception", [(k, v) for k, v in TEST_CASES_NEGATIVE.items()]) -@patch("process_sd.download_sd_by_appver") -def test_sd_negative(mock_download_sd, test_case_name, expected_exception): - env = Environment(str(Path(OUTPUT_DIR, test_case_name)), "cluster-01", "env-01") - do_prerequisites(SD, TEST_SD_DIR, OUTPUT_DIR, test_case_name, env, test_suits_map) - logger.info(f"======TEST HANDLE_SD_ARTIFACT_NEGATIVE: {test_case_name}======") - logger.info(f"Starting SD test:" - f"\n\tTest case: {test_case_name}") - - sd_data, sd_source_type, sd_version, sd_delta, sd_merge_mode = load_test_pipeline_sd_data(TEST_SD_DIR, test_case_name) - - file_path = Path(TEST_SD_DIR, test_case_name, f"mock_sd.json") - sd_data = openJson(file_path) - mock_download_sd.return_value = sd_data - - with pytest.raises(expected_exception): +FEATURE_TEST_DIR = "test_handle_sd" + + +class TestSdProcessArtifact(BaseTest): + + def setup_method(self): + self.env_name = "env-01" + self.cluster = "cluster-01" + self.full_env_name = f"{self.cluster}/{self.env_name}" + + self.set_ci_project_dir(FEATURE_TEST_DIR) + self.test_data_dir = self.test_data_dir / FEATURE_TEST_DIR + self.output_dir = self.output_dir / FEATURE_TEST_DIR + + os.environ["FULL_ENV_NAME"] = self.full_env_name + os.environ["ENV_NAME"] = self.env_name + os.environ["CLUSTER_NAME"] = self.cluster + + @pytest.mark.parametrize("test_case_name", TEST_CASES_POSITIVE) + @patch("process_sd.download_sd_by_appver") + def test_sd_positive(self, mock_download_sd, test_case_name): + env = Environment(str(Path(self.output_dir, test_case_name)), self.cluster, self.env_name) + do_prerequisites(SD_FILE_NAME, self.test_data_dir, self.output_dir, test_case_name, env, test_suits_map) + logger.info(f"======TEST HANDLE_SD_ARTIFACT_POSITIVE: {test_case_name}======") + logger.info(f"Starting SD test:" + f"\n\tTest case: {test_case_name}") + + sd_data, sd_source_type, sd_version, sd_delta, sd_merge_mode = load_test_pipeline_sd_data(self.test_data_dir, + test_case_name) + + file_path = Path(self.test_data_dir, test_case_name, "mock_sd.json") + sd_data = openJson(file_path) + mock_download_sd.return_value = sd_data + handle_sd(env, sd_source_type, sd_version, sd_data, sd_delta, sd_merge_mode) + actual_dir = os.path.join(env.env_path, "Inventory", "solution-descriptor") + + assert_sd_contents(self.test_data_dir, self.output_dir, test_case_name, actual_dir, test_suits_map) + logger.info(f"=====SUCCESS - {test_case_name}======") + + @pytest.mark.parametrize("test_case_name,expected_exception", [(k, v) for k, v in TEST_CASES_NEGATIVE.items()]) + @patch("process_sd.download_sd_by_appver") + def test_sd_negative(self, mock_download_sd, test_case_name, expected_exception): + + env = Environment(str(Path(self.output_dir, test_case_name)), self.cluster, self.env_name) + do_prerequisites(SD_FILE_NAME, self.test_data_dir, self.output_dir, test_case_name, env, test_suits_map) + logger.info(f"======TEST HANDLE_SD_ARTIFACT_NEGATIVE: {test_case_name}======") + logger.info(f"Starting SD test:" + f"\n\tTest case: {test_case_name}") + + sd_data, sd_source_type, sd_version, sd_delta, sd_merge_mode = load_test_pipeline_sd_data(self.test_data_dir, + test_case_name) + + file_path = Path(self.test_data_dir, test_case_name, "mock_sd.json") + sd_data = openJson(file_path) + mock_download_sd.return_value = sd_data + + with pytest.raises(expected_exception): + handle_sd(env, sd_source_type, sd_version, sd_data, sd_delta, sd_merge_mode) - logger.info(f"=====SUCCESS - {test_case_name}======") + logger.info(f"=====SUCCESS - {test_case_name}======") diff --git a/scripts/build_env/tests/sd/test_process_sd_local.py b/scripts/build_env/tests/sd/test_process_sd_local.py index d6a4a8082..4fa855bba 100644 --- a/scripts/build_env/tests/sd/test_process_sd_local.py +++ b/scripts/build_env/tests/sd/test_process_sd_local.py @@ -1,8 +1,11 @@ import os +from pathlib import Path import pytest -from ruamel.yaml import YAML +from envgenehelper import logger, SD_FILE_NAME +from envgenehelper.env_helper import Environment +from scripts.build_env.tests.base_test import BaseTest from test_sd_helpers import do_prerequisites, assert_sd_contents, load_test_pipeline_sd_data os.environ['ENVIRONMENT_NAME'] = "temporary" @@ -10,10 +13,7 @@ os.environ['CI_PROJECT_DIR'] = "temporary" from process_sd import handle_sd -from envgenehelper import * -from envgenehelper.env_helper import Environment -yaml = YAML() TEST_CASES_POSITIVE = [ "TC-001-002", "TC-001-004", @@ -34,23 +34,36 @@ "replace": ["TC-001-008", "TC-001-006"] } -TEST_SD_DIR = Path(getAbsPath("../../test_data/test_handle_sd")) -OUTPUT_DIR = getAbsPath("../../tmp/test_handle_sd") -SD = "sd.yaml" +FEATURE_TEST_DIR = "test_handle_sd" + + +class TestSdProcessArtifact(BaseTest): + def setup_method(self): + self.env_name = "env-01" + self.cluster = "cluster-01" + self.full_env_name = f"{self.cluster}/{self.env_name}" + + self.set_ci_project_dir(FEATURE_TEST_DIR) + self.test_data_dir = self.test_data_dir / FEATURE_TEST_DIR + self.output_dir = self.output_dir / FEATURE_TEST_DIR + + os.environ["FULL_ENV_NAME"] = self.full_env_name + os.environ["ENV_NAME"] = self.env_name + os.environ["CLUSTER_NAME"] = self.cluster + @pytest.mark.parametrize("test_case_name", TEST_CASES_POSITIVE) + def test_sd_positive(self, test_case_name): + env = Environment(str(Path(self.output_dir, test_case_name)), "cluster-01", "env-01") + do_prerequisites(SD_FILE_NAME, self.test_data_dir, self.output_dir, test_case_name, env, test_suits_map) + logger.info(f"======TEST HANDLE_SD_LOCAL_POSITIVE: {test_case_name}======") + logger.info(f"Starting SD test:" + f"\n\tTest case: {test_case_name}") -@pytest.mark.parametrize("test_case_name", TEST_CASES_POSITIVE) -def test_sd_positive(test_case_name): - env = Environment(str(Path(OUTPUT_DIR, test_case_name)), "cluster-01", "env-01") - do_prerequisites(SD, TEST_SD_DIR, OUTPUT_DIR, test_case_name, env, test_suits_map) - logger.info(f"======TEST HANDLE_SD_LOCAL_POSITIVE: {test_case_name}======") - logger.info(f"Starting SD test:" - f"\n\tTest case: {test_case_name}") + sd_data, sd_source_type, sd_version, sd_delta, sd_merge_mode = load_test_pipeline_sd_data(self.test_data_dir, + test_case_name) - sd_data, sd_source_type, sd_version, sd_delta, sd_merge_mode = load_test_pipeline_sd_data(TEST_SD_DIR, test_case_name) - - handle_sd(env, sd_source_type, sd_version, sd_data, sd_delta, sd_merge_mode) - actual_dir = os.path.join(env.env_path, "Inventory", "solution-descriptor") + handle_sd(env, sd_source_type, sd_version, sd_data, sd_delta, sd_merge_mode) + actual_dir = os.path.join(env.env_path, "Inventory", "solution-descriptor") - assert_sd_contents(TEST_SD_DIR, OUTPUT_DIR, test_case_name, actual_dir, test_suits_map) - logger.info(f"=====SUCCESS - {test_case_name}======") + assert_sd_contents(self.test_data_dir, self.output_dir, test_case_name, actual_dir, test_suits_map) + logger.info(f"=====SUCCESS - {test_case_name}======") diff --git a/scripts/build_env/tests/sd/test_sd_helpers.py b/scripts/build_env/tests/sd/test_sd_helpers.py index c90ea8f35..b97405fdc 100644 --- a/scripts/build_env/tests/sd/test_sd_helpers.py +++ b/scripts/build_env/tests/sd/test_sd_helpers.py @@ -3,6 +3,7 @@ from envgenehelper import openYaml, writeYamlToFile from envgenehelper.test_helpers import TestHelpers + def load_test_pipeline_sd_data(test_sd_dir, test_case_name): file_path = Path(test_sd_dir, test_case_name, f"{test_case_name}.yaml") test_data = openYaml(file_path) @@ -27,6 +28,7 @@ def do_prerequisites(sd, test_sd_dir, output_dir, test_case_name, env, test_suit elif test_case_name in test_suits_map["extended"]: writeYamlToFile(target_sd_dir.joinpath(sd), openYaml(pr_dir.joinpath("extended").joinpath(sd))) + def assert_sd_contents(test_sd_dir, output_dir, test_case_name, actual_dir, test_suits_map): er_dir = test_sd_dir.joinpath("ER") diff --git a/scripts/utils/entrypoint.sh b/scripts/utils/run_effective_set_cli.sh old mode 100755 new mode 100644 similarity index 100% rename from scripts/utils/entrypoint.sh rename to scripts/utils/run_effective_set_cli.sh diff --git a/test_data/test_handle_sd/ER/basic_not_first/delta_sd.yaml b/test_data/test_handle_sd/ER/basic_not_first/delta_sd.yaml deleted file mode 100644 index 9283f3e83..000000000 --- a/test_data/test_handle_sd/ER/basic_not_first/delta_sd.yaml +++ /dev/null @@ -1,5 +0,0 @@ -applications: - - version: postgres-services:1.32.6 - deployPostfix: postgresql - - version: postgres:1.32.7 - deployPostfix: postgresql-dbaas diff --git a/test_data/test_handle_sd/ER/exclude/delta_sd.yaml b/test_data/test_handle_sd/ER/exclude/delta_sd.yaml deleted file mode 100644 index 59938d6ec..000000000 --- a/test_data/test_handle_sd/ER/exclude/delta_sd.yaml +++ /dev/null @@ -1,5 +0,0 @@ -applications: - - version: postgres-services:1.32.6 - deployPostfix: postgresql - - version: postgres:1.32.6 - deployPostfix: postgresql-dbaas diff --git a/test_data/test_handle_sd/ER/extended/delta_sd.yaml b/test_data/test_handle_sd/ER/extended/delta_sd.yaml deleted file mode 100644 index 9fd14d0dd..000000000 --- a/test_data/test_handle_sd/ER/extended/delta_sd.yaml +++ /dev/null @@ -1,19 +0,0 @@ -version: 2.1 -type: solutionDeploy -deployMode: composite -applications: - - version: postgres:1.32.6 - deployPostfix: postgresql - - version: mysql:8.0.34 - deployPostfix: mysql-dbaas - - version: mongodb:7.0.3 - deployPostfix: mongo-db -deployGraph: - - chunkName: wave1 - apps: - - postgres:postgresql - - mysql:mysql-dbaas - - mongodb:mongo-db - - chunkName: wave2 - apps: - - redis:redis-cache