From 2be33862586211e6d184252e2e295c6daabca838 Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Thu, 30 Apr 2026 21:15:12 +0500 Subject: [PATCH 01/24] feat: Partial Effective Set generation --- build_pipegene/scripts/effective_set_job.py | 38 +++++++++---------- build_pipegene/scripts/process_sd_job.py | 14 +++---- .../envgene/envgenehelper/business_helper.py | 15 +++++++- scripts/build_env/process_sd.py | 34 +++++++++++++---- 4 files changed, 64 insertions(+), 37 deletions(-) diff --git a/build_pipegene/scripts/effective_set_job.py b/build_pipegene/scripts/effective_set_job.py index 2db54fecf..658f0dfd5 100644 --- a/build_pipegene/scripts/effective_set_job.py +++ b/build_pipegene/scripts/effective_set_job.py @@ -2,11 +2,12 @@ from os import getenv, environ from pathlib import Path +from envgenehelper import cleanup_targets +from envgenehelper import logger from gcip import WhenStatement, Need -from envgenehelper import logger, get_sboms_dir -from envgenehelper import cleanup_targets from pipeline_helper import job_instance +from build_env.process_sd import resolve_sd_path def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluster_name, params): @@ -16,26 +17,21 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste app_reg_defs_job = params["APP_REG_DEFS_JOB"] artifact_app_defs_path = params["APP_DEFS_PATH"] artifact_reg_defs_path = params["REG_DEFS_PATH"] - sd_version = params["SD_VERSION"] - sd_data = params["SD_DATA"] deployment_id = params["DEPLOYMENT_SESSION_ID"] effective_set_config = params["EFFECTIVE_SET_CONFIG"] if "CUSTOM_PARAMS" in params: custom_params = params["CUSTOM_PARAMS"] is_local_app_def = artifact_app_defs_path and artifact_reg_defs_path and app_reg_defs_job - - base_dir = getenv('CI_PROJECT_DIR') - - sd_path = Path(f'{base_dir}/environments/{full_env_name}/Inventory/solution-descriptor/sd.yaml') + sd_path = resolve_sd_path() # TODO it is necessary to remove unnecessary calls, leave only script calls in such jobs! bad for gsf delivery script = [ - #Overriding sd_path to pick the correct value for CI_PROJECT_DIR + # Overriding sd_path to pick the correct value for CI_PROJECT_DIR f'base_env_path="$CI_PROJECT_DIR/environments/{full_env_name}";', 'app_defs_path="$base_env_path/AppDefs";', 'reg_defs_path="$base_env_path/RegDefs";', 'sboms_path="$CI_PROJECT_DIR/sboms";', - 'sd_path="$base_env_path/Inventory/solution-descriptor/sd.yaml";', + f'sd_path="$base_env_path/Inventory/solution-descriptor/{sd_path.name}";', # cert handling for java 'mkdir -p ${CI_PROJECT_DIR}/configuration/certs/', 'if [ -f /default_cert.pem ]; then cp /default_cert.pem "${CI_PROJECT_DIR}/configuration/certs/"; fi', @@ -56,15 +52,8 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste effective_set_config_dict = {} if effective_set_config: effective_set_config_dict = json.loads(effective_set_config) - - effective_set_version = effective_set_config_dict.get("version") or "v2.0" - full_sd_exists = sd_path.is_file() - sd_data = bool(sd_data) or bool(sd_version) - - if not (full_sd_exists and sd_data) and effective_set_version.lower() == "v1.0": - raise ValueError("Feature generation effective set for pipeline and topology context is not supported for v1.0") - - if full_sd_exists or sd_data: + validate_topology_context_mode(effective_set_config_dict, full_env_name, params) + if sd_path.is_file(): cmdb_cli_cmd_call.extend([ "--registries=${CI_PROJECT_DIR}/configuration/registry.yml", "--sboms-path=$sboms_path", @@ -114,7 +103,7 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste needs.append(Need(job=app_reg_defs_job, pipeline=real_ci_pipe_id, artifacts=True)) environ['CI_PIPELINE_ID'] = real_ci_pipe_id generate_effective_set_job = job_instance(params=generate_effective_set_params, needs=needs, - vars=generate_effective_set_vars) + vars=generate_effective_set_vars) effective_set_expiry = effective_set_config_dict.get("effective_set_expiry") or "1 hour" logger.info(f"effective set expiry value '{effective_set_expiry}'") @@ -124,3 +113,12 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste pipeline.add_children(generate_effective_set_job) return generate_effective_set_job + + +def validate_topology_context_mode(effective_set_config_dict, full_env_name, params): + effective_set_version = effective_set_config_dict.get("version") or "v2.0" + sd = bool(params["SD_DATA"]) or bool(params["SD_VERSION"]) + full_sd_path = Path(f'{getenv('CI_PROJECT_DIR')}/environments/{full_env_name}/Inventory/solution-descriptor/sd.yaml') + # effective set generation in version 1.0 does not support no SBOMs mode + if not (full_sd_path.exists() and sd) and effective_set_version.lower() == "v1.0": + raise ValueError("Feature generation effective set for pipeline and topology context is not supported for v1.0") diff --git a/build_pipegene/scripts/process_sd_job.py b/build_pipegene/scripts/process_sd_job.py index 641671f19..758cfe545 100644 --- a/build_pipegene/scripts/process_sd_job.py +++ b/build_pipegene/scripts/process_sd_job.py @@ -1,14 +1,12 @@ -from os import getenv - +from envgenehelper import logger from gcip import WhenStatement -from envgenehelper import logger from pipeline_helper import job_instance def prepare_process_sd(pipeline, full_env, environment_name, cluster_name): logger.info(f'Prepare process_sd job for {full_env}') - + script = [ f'base_env_path="$CI_PROJECT_DIR/environments/{full_env}";', 'app_defs_path="$base_env_path/AppDefs";', @@ -28,12 +26,12 @@ def prepare_process_sd(pipeline, full_env, environment_name, cluster_name): process_sd_set_vars = { "CLUSTER_NAME": cluster_name, "ENVIRONMENT_NAME": environment_name, - "ENV_NAME": environment_name, "INSTANCES_DIR": "${CI_PROJECT_DIR}/environments", + "FULL_ENV_NAME": full_env, } process_sd_job = job_instance(params=process_sd_set_params, vars=process_sd_set_vars) - process_sd_job.artifacts.when = WhenStatement.ALWAYS + process_sd_job.artifacts.when = WhenStatement.ALWAYS pipeline.add_children(process_sd_job) - - return process_sd_job \ No newline at end of file + + return process_sd_job diff --git a/python/envgene/envgenehelper/business_helper.py b/python/envgene/envgenehelper/business_helper.py index 86879dd10..4c53bae4a 100644 --- a/python/envgene/envgenehelper/business_helper.py +++ b/python/envgene/envgenehelper/business_helper.py @@ -27,6 +27,8 @@ DEFAULT_PASSPORT_NAME = "passport" DEFAULT_PASSPORT_DIR_NAME = "cloud-passport" INV_GEN_CREDS_PATH = "Inventory/credentials/inventory_generation_creds.yml" +SD_FILE_NAME = "sd.yaml" +DELTA_SD_FILE_NAME = "delta_sd.yaml" TEMPLATE_DIR_PATTERN = re.compile(r'/from_(\w+_)?template/') @@ -169,7 +171,7 @@ def getTemplateArtifactName(env_definition_yaml): return gav["artifact_id"] -def getEnvDefinition(env_dir = None): +def getEnvDefinition(env_dir=None): env_dir = env_dir or get_current_env_dir_from_env_vars() env_definition_path = getEnvDefinitionPath(env_dir) if not check_file_exists(env_definition_path): @@ -371,11 +373,13 @@ def find_cloud_name_from_passport(source_env_dir, all_instances_dir): else: return "" + class NamespaceRole(StrEnum): COMMON = auto() ORIGIN = auto() PEER = auto() + def get_namespace_role(ns_name: str, bgd_object: dict | None = None) -> NamespaceRole: if not bgd_object: bgd_object = get_bgd_object() @@ -387,6 +391,7 @@ def get_namespace_role(ns_name: str, bgd_object: dict | None = None) -> Namespac return NamespaceRole.PEER return NamespaceRole.COMMON + @dataclass class NamespaceFile: path: Path @@ -409,6 +414,7 @@ def get_namespaces_path(env_dir: Path | None = None) -> Path: logger.debug(namespaces_path) return namespaces_path + def get_bgd_path(env_dir: Path | None = None) -> Path: env_dir = env_dir or get_current_env_dir_from_env_vars() bgd_path = env_dir.joinpath('bg_domain.yml') @@ -422,6 +428,7 @@ def get_bgd_object(env_dir: Path | None = None) -> CommentedMap: logger.debug(bgd_object) return bgd_object + def get_namespaces(env_dir: Path | None = None) -> list[NamespaceFile]: namespaces_path = get_namespaces_path(env_dir) if not check_dir_exists(str(namespaces_path)): @@ -432,6 +439,7 @@ def get_namespaces(env_dir: Path | None = None) -> list[NamespaceFile]: logger.debug(namespaces) return namespaces + def get_template_dirs(base_dir: str | None = None) -> dict[NamespaceRole, str]: base_dir = base_dir if base_dir else getenv_with_error('CI_PROJECT_DIR') result = {} @@ -444,9 +452,14 @@ def get_template_dirs(base_dir: str | None = None) -> dict[NamespaceRole, str]: result[NamespaceRole.PEER] = peer_template_path return result + def is_from_template_dir(file_path: str) -> bool: return bool(TEMPLATE_DIR_PATTERN.search(file_path)) def get_sboms_dir(work_dir) -> Path: return Path(work_dir) / "sboms" + + +def get_sd_dir() -> Path: + return Path(f'{get_current_env_dir_from_env_vars()}/{INVENTORY_DIR_NAME}/solution-descriptor/') diff --git a/scripts/build_env/process_sd.py b/scripts/build_env/process_sd.py index 47c0d103a..701888b61 100644 --- a/scripts/build_env/process_sd.py +++ b/scripts/build_env/process_sd.py @@ -17,6 +17,8 @@ from envgenehelper.plugin_engine import PluginEngine from envgenehelper.sd_merge_helper import basic_merge_multiple from envgenehelper.collections_helper import split_multi_value_param +from envgenehelper import get_sd_dir, SD_FILE_NAME, DELTA_SD_FILE_NAME +from typing_extensions import deprecated class MergeType(Enum): @@ -88,7 +90,7 @@ def handle_deploy_postfix_namespace_transformation(sd_data: dict, namespace_dict def prepare_vars_and_run_sd_handling(): base_dir = getenv_and_log('CI_PROJECT_DIR') - env_name = getenv_and_log('ENV_NAME') + env_name = getenv_and_log('ENVIRONMENT_NAME') cluster = getenv_and_log('CLUSTER_NAME') env = Environment(base_dir, cluster, env_name) @@ -134,7 +136,7 @@ def build_namespace_dict(env) -> dict: def merge_sd(sd_path: Path, sd_data, merge_func): logger.info(f"Final destination! - {sd_path}") full_sd_yaml = helper.openYaml(sd_path) - logger.info(f"full_sd.yaml before merge: {full_sd_yaml}") + logger.info(f"Full sd before merge: {full_sd_yaml}") helper.check_dir_exist_and_create(sd_path.parent) result = merge_func(full_sd_yaml, sd_data) helper.writeYamlToFile(sd_path, result) @@ -144,6 +146,7 @@ def merge_sd(sd_path: Path, sd_data, merge_func): def calculate_merge_mode(sd_merge_mode, sd_delta) -> MergeType: if sd_merge_mode is not None: effective_merge_mode = MergeType.from_value(sd_merge_mode) + # sd_delta var is deprecated elif sd_delta == "true": effective_merge_mode = MergeType.EXTENDED logger.info( @@ -158,6 +161,7 @@ def calculate_merge_mode(sd_merge_mode, sd_delta) -> MergeType: return effective_merge_mode +@deprecated def calculate_sd_delta(sd_delta): logger.info(f"printing sd_delta before {sd_delta}") if sd_delta is not None and str(sd_delta).strip() != "": @@ -188,9 +192,23 @@ def multiply_sds_to_single(sds_data, effective_merge_mode): return full_sd_from_pipe -def handle_sd(env, sd_source_type, sd_version, sd_data, sd_delta, sd_merge_mode): - base_sd_path = Path(f'{env.env_path}/Inventory/solution-descriptor/') +def resolve_sd_path() -> Path: + sd_delta = getenv('SD_DELTA') + sd_merge_mode = getenv("SD_REPO_MERGE_MODE") + base_sd_path = get_sd_dir() + merge_mode = calculate_merge_mode(sd_merge_mode, sd_delta) + full_sd_path = base_sd_path.joinpath(SD_FILE_NAME) + sd_version = getenv("SD_VERSION") + sd_data = getenv("SD_DATA") + sd_input = bool(sd_data) or bool(sd_version) + if merge_mode == MergeType.REPLACE or (not sd_input and full_sd_path.is_file()): + return full_sd_path + elif merge_mode == MergeType.BASIC or merge_mode == MergeType.EXTENDED: + return base_sd_path.joinpath(DELTA_SD_FILE_NAME) + +def handle_sd(env, sd_source_type, sd_version, sd_data, sd_delta, sd_merge_mode): + base_sd_path = get_sd_dir() sd_delta = calculate_sd_delta(sd_delta) effective_merge_mode = calculate_merge_mode(sd_merge_mode, sd_delta) @@ -238,13 +256,13 @@ def extract_sds_from_json(env, base_sd_path: Path, sd_data, effective_merge_mode full_sd_from_pipe = multiply_sds_to_single(transformed_data, effective_merge_mode) validate_applications(full_sd_from_pipe, effective_merge_mode) - sd_path = base_sd_path.joinpath("sd.yaml") - sd_delta_path = base_sd_path.joinpath("delta_sd.yaml") + sd_path = base_sd_path.joinpath(SD_FILE_NAME) + sd_delta_path = base_sd_path.joinpath(DELTA_SD_FILE_NAME) if effective_merge_mode == MergeType.REPLACE: logger.info("Inside replace") if helper.check_file_exists(sd_path): full_sd_yaml = helper.openYaml(sd_path) - logger.info(f"full_sd.yaml before replacement: {json.dumps(full_sd_yaml, indent=2)}") + logger.info(f"Full sd before replacement: {json.dumps(full_sd_yaml, indent=2)}") else: logger.info("No existing SD found at destination. Proceeding to write new SD.") helper.check_dir_exist_and_create(path.dirname(sd_path)) @@ -306,7 +324,7 @@ def download_sd_by_appver(app_name: str, version: str, plugins: PluginEngine) -> artifact_info = asyncio.run( artifact.check_artifact_async(app_def, artifact.FileExtension.JSON, version, - auth_headers=auth_headers)) + auth_headers=auth_headers)) if not artifact_info: raise ValueError( f'Solution descriptor content was not received for {app_name}:{version}') From 486963de4e3ce8ff2335c53297788003b15f26e4 Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Thu, 30 Apr 2026 21:19:50 +0500 Subject: [PATCH 02/24] feat: Partial Effective Set generation --- build_pipegene/scripts/effective_set_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_pipegene/scripts/effective_set_job.py b/build_pipegene/scripts/effective_set_job.py index 658f0dfd5..cad98960a 100644 --- a/build_pipegene/scripts/effective_set_job.py +++ b/build_pipegene/scripts/effective_set_job.py @@ -7,7 +7,7 @@ from gcip import WhenStatement, Need from pipeline_helper import job_instance -from build_env.process_sd import resolve_sd_path +from process_sd import resolve_sd_path def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluster_name, params): From 0d1d28eae4c564cb122712016ca38d2d0334c83c Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Sun, 3 May 2026 14:08:34 +0500 Subject: [PATCH 03/24] feat: Partial Effective Set generation --- build_pipegene/scripts/effective_set_job.py | 3 +- python/envgene/envgenehelper/__init__.py | 4 +- .../{sd_merge_helper.py => sd_helper.py} | 56 +++++++++++++++- scripts/build_env/process_sd.py | 64 ++----------------- 4 files changed, 62 insertions(+), 65 deletions(-) rename python/envgene/envgenehelper/{sd_merge_helper.py => sd_helper.py} (77%) diff --git a/build_pipegene/scripts/effective_set_job.py b/build_pipegene/scripts/effective_set_job.py index cad98960a..e97143ceb 100644 --- a/build_pipegene/scripts/effective_set_job.py +++ b/build_pipegene/scripts/effective_set_job.py @@ -2,12 +2,11 @@ from os import getenv, environ from pathlib import Path -from envgenehelper import cleanup_targets +from envgenehelper import cleanup_targets, resolve_sd_path from envgenehelper import logger from gcip import WhenStatement, Need from pipeline_helper import job_instance -from process_sd import resolve_sd_path def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluster_name, params): diff --git a/python/envgene/envgenehelper/__init__.py b/python/envgene/envgenehelper/__init__.py index cace6f169..275d33ac5 100644 --- a/python/envgene/envgenehelper/__init__.py +++ b/python/envgene/envgenehelper/__init__.py @@ -7,7 +7,7 @@ from .collections_helper import * from .logger import logger from .creds_helper import * -from .sd_merge_helper import * +from .sd_helper import * from .yaml_validator import checkByWhiteList, checkByBlackList, checkSchemaValidationFailed, getSchemaValidationErrorMessage from .crypt import decrypt_file, encrypt_file, decrypt_all_cred_files_for_env, encrypt_all_cred_files_for_env, is_encrypted -from .constants import cleanup_targets \ No newline at end of file +from .constants import cleanup_targets diff --git a/python/envgene/envgenehelper/sd_merge_helper.py b/python/envgene/envgenehelper/sd_helper.py similarity index 77% rename from python/envgene/envgenehelper/sd_merge_helper.py rename to python/envgene/envgenehelper/sd_helper.py index 173498034..dd476b74a 100644 --- a/python/envgene/envgenehelper/sd_merge_helper.py +++ b/python/envgene/envgenehelper/sd_helper.py @@ -80,7 +80,8 @@ def add_app(entry, apps: list) -> int: apps.append(entry) return 1 -#TODO : notation is supported only for extended merge for now, but later have to be removed + +# TODO : notation is supported only for extended merge for now, but later have to be removed def extended_merge(full_sd, delta_sd): # Merges delta SD into full SD by updating or adding matching apps, ensuring deployGraph consistency logger.info("Inside extended_merge") @@ -227,3 +228,56 @@ def basic_exclusion_merge(full_sd, delta_sd): logger.info(f"Warning: New application '{get_app_name_sd(d_app)}' ignored (not present in Full SD)") return {"applications": result_apps} + + +class MergeType(Enum): + EXTENDED = "extended-merge" + REPLACE = "replace" + BASIC = "basic-merge" + BASIC_EXCLUSION = "basic-exclusion-merge" + + @classmethod + def from_value(cls, value: str): + if not isinstance(value, str): + raise ValueError(f"SD_REPO_MERGE_MODE value: '{value}' cannot be non-string") + value_lower = value.strip().lower() + for member in cls: + if member.value == value_lower: + return member + valid_values = [member.value for member in cls] + raise ValueError( + f"Invalid SD_REPO_MERGE_MODE: '{value}'. Valid values are: {valid_values}" + ) + + +def calculate_merge_mode(sd_merge_mode, sd_delta) -> MergeType: + if sd_merge_mode is not None: + effective_merge_mode = MergeType.from_value(sd_merge_mode) + # sd_delta var is deprecated + elif sd_delta == "true": + effective_merge_mode = MergeType.EXTENDED + logger.info( + f"SD_REPO_MERGE_MODE not passed. Calculated based on SD_DELTA={sd_delta}: {effective_merge_mode.value}") + elif sd_delta == "false": + effective_merge_mode = MergeType.REPLACE + logger.info( + f"SD_REPO_MERGE_MODE not passed. Calculated based on SD_DELTA={sd_delta}: {effective_merge_mode.value}") + else: + effective_merge_mode = MergeType.BASIC + logger.info(f"SD_REPO_MERGE_MODE not passed. Default value: {effective_merge_mode.value}") + return effective_merge_mode + + +def resolve_sd_path() -> Path: + sd_delta = getenv('SD_DELTA') + sd_merge_mode = getenv("SD_REPO_MERGE_MODE") + base_sd_path = get_sd_dir() + merge_mode = calculate_merge_mode(sd_merge_mode, sd_delta) + full_sd_path = base_sd_path.joinpath(SD_FILE_NAME) + sd_version = getenv("SD_VERSION") + sd_data = getenv("SD_DATA") + sd_input = bool(sd_data) or bool(sd_version) + if merge_mode == MergeType.REPLACE or (not sd_input and full_sd_path.is_file()): + return full_sd_path + elif merge_mode == MergeType.BASIC or merge_mode == MergeType.EXTENDED: + return base_sd_path.joinpath(DELTA_SD_FILE_NAME) diff --git a/scripts/build_env/process_sd.py b/scripts/build_env/process_sd.py index 701888b61..d756d643a 100644 --- a/scripts/build_env/process_sd.py +++ b/scripts/build_env/process_sd.py @@ -1,46 +1,23 @@ import asyncio import json import os -from enum import Enum from os import path, getenv from pathlib import Path -import yaml - import envgenehelper as helper +import yaml from artifact_searcher import artifact from artifact_searcher.utils import models as artifact_models +from envgenehelper import get_sd_dir, SD_FILE_NAME, DELTA_SD_FILE_NAME from envgenehelper.business_helper import getenv_and_log, getenv_with_error +from envgenehelper.collections_helper import split_multi_value_param from envgenehelper.env_helper import Environment from envgenehelper.file_helper import identify_yaml_extension from envgenehelper.logger import logger from envgenehelper.plugin_engine import PluginEngine -from envgenehelper.sd_merge_helper import basic_merge_multiple -from envgenehelper.collections_helper import split_multi_value_param -from envgenehelper import get_sd_dir, SD_FILE_NAME, DELTA_SD_FILE_NAME +from envgenehelper.sd_helper import basic_merge_multiple, MergeType, calculate_merge_mode from typing_extensions import deprecated - -class MergeType(Enum): - EXTENDED = "extended-merge" - REPLACE = "replace" - BASIC = "basic-merge" - BASIC_EXCLUSION = "basic-exclusion-merge" - - @classmethod - def from_value(cls, value: str): - if not isinstance(value, str): - raise ValueError(f"SD_REPO_MERGE_MODE value: '{value}' cannot be non-string") - value_lower = value.strip().lower() - for member in cls: - if member.value == value_lower: - return member - valid_values = [member.value for member in cls] - raise ValueError( - f"Invalid SD_REPO_MERGE_MODE: '{value}'. Valid values are: {valid_values}" - ) - - MERGE_METHODS = { MergeType.BASIC: helper.basic_merge, MergeType.BASIC_EXCLUSION: helper.basic_exclusion_merge, @@ -143,24 +120,6 @@ def merge_sd(sd_path: Path, sd_data, merge_func): logger.info(f"Merged data into Target Path! - {result}") -def calculate_merge_mode(sd_merge_mode, sd_delta) -> MergeType: - if sd_merge_mode is not None: - effective_merge_mode = MergeType.from_value(sd_merge_mode) - # sd_delta var is deprecated - elif sd_delta == "true": - effective_merge_mode = MergeType.EXTENDED - logger.info( - f"SD_REPO_MERGE_MODE not passed. Calculated based on SD_DELTA={sd_delta}: {effective_merge_mode.value}") - elif sd_delta == "false": - effective_merge_mode = MergeType.REPLACE - logger.info( - f"SD_REPO_MERGE_MODE not passed. Calculated based on SD_DELTA={sd_delta}: {effective_merge_mode.value}") - else: - effective_merge_mode = MergeType.BASIC - logger.info(f"SD_REPO_MERGE_MODE not passed. Default value: {effective_merge_mode.value}") - return effective_merge_mode - - @deprecated def calculate_sd_delta(sd_delta): logger.info(f"printing sd_delta before {sd_delta}") @@ -192,21 +151,6 @@ def multiply_sds_to_single(sds_data, effective_merge_mode): return full_sd_from_pipe -def resolve_sd_path() -> Path: - sd_delta = getenv('SD_DELTA') - sd_merge_mode = getenv("SD_REPO_MERGE_MODE") - base_sd_path = get_sd_dir() - merge_mode = calculate_merge_mode(sd_merge_mode, sd_delta) - full_sd_path = base_sd_path.joinpath(SD_FILE_NAME) - sd_version = getenv("SD_VERSION") - sd_data = getenv("SD_DATA") - sd_input = bool(sd_data) or bool(sd_version) - if merge_mode == MergeType.REPLACE or (not sd_input and full_sd_path.is_file()): - return full_sd_path - elif merge_mode == MergeType.BASIC or merge_mode == MergeType.EXTENDED: - return base_sd_path.joinpath(DELTA_SD_FILE_NAME) - - def handle_sd(env, sd_source_type, sd_version, sd_data, sd_delta, sd_merge_mode): base_sd_path = get_sd_dir() sd_delta = calculate_sd_delta(sd_delta) From cd87cd5dbf81ebbf6c66e18e0526de42baca28c7 Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Sun, 3 May 2026 14:21:20 +0500 Subject: [PATCH 04/24] feat: Partial Effective Set generation --- build_pipegene/scripts/effective_set_job.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_pipegene/scripts/effective_set_job.py b/build_pipegene/scripts/effective_set_job.py index e97143ceb..ca81f793f 100644 --- a/build_pipegene/scripts/effective_set_job.py +++ b/build_pipegene/scripts/effective_set_job.py @@ -89,7 +89,8 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste "ENV_NAME": env_name, "INSTANCES_DIR": "${CI_PROJECT_DIR}/environments", "effective_set_generator_image": "$effective_set_generator_image", - "EXCLUDE_CLEANUP_TARGETS": " ".join(cleanup_targets) + "EXCLUDE_CLEANUP_TARGETS": " ".join(cleanup_targets), + "FULL_ENV_NAME": full_env_name, } needs = [] From 87bd8dbb1660d0e64188551ed99707b08e243d8e Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Mon, 4 May 2026 16:27:45 +0500 Subject: [PATCH 05/24] feat: refactor effective set job --- .../scripts/effective_set_entrypoint.py | 81 +++++++++++++++++++ build_effective_set_generator/scripts/main.py | 26 ------ .../scripts/sboms_retention_policy.py | 4 - build_pipegene/scripts/effective_set_job.py | 68 +--------------- ...entrypoint.sh => run_effective_set_cli.sh} | 0 5 files changed, 82 insertions(+), 97 deletions(-) create mode 100644 build_effective_set_generator/scripts/effective_set_entrypoint.py delete mode 100644 build_effective_set_generator/scripts/main.py rename scripts/utils/{entrypoint.sh => run_effective_set_cli.sh} (100%) mode change 100755 => 100644 diff --git a/build_effective_set_generator/scripts/effective_set_entrypoint.py b/build_effective_set_generator/scripts/effective_set_entrypoint.py new file mode 100644 index 000000000..158f49015 --- /dev/null +++ b/build_effective_set_generator/scripts/effective_set_entrypoint.py @@ -0,0 +1,81 @@ +import json +import subprocess +from os import getenv +from pathlib import Path + +from envgenehelper import decrypt_all_cred_files_for_env, copy_path, validate_creds, openJson, \ + encrypt_all_cred_files_for_env, resolve_sd_path, logger + +from handle_effective_set_config import handle_effective_set_config +from sboms_retention_policy import sboms_retention_policy + + +def effective_set_entrypoint(): + full_env_name = getenv("FULL_ENV_NAME") + ci_project_dir = getenv("CI_PROJECT_DIR") + work_dir = ci_project_dir / "environments" / full_env_name + + decrypt_all_cred_files_for_env() + + artifact_app_defs_path = getenv("APP_DEFS_PATH") + artifact_reg_defs_path = getenv("REG_DEFS_PATH") + app_reg_defs_job = getenv("APP_REG_DEFS_JOB") + is_local_app_def = artifact_app_defs_path and artifact_reg_defs_path and app_reg_defs_job + if is_local_app_def: + app_defs_path = work_dir / "AppDefs" + reg_defs_path = work_dir / "RegDefs" + copy_path(artifact_app_defs_path, app_defs_path) + copy_path(artifact_reg_defs_path, reg_defs_path) + + validate_creds() + sboms_retention_policy() + + cmdb_cli_cmd_call = [ + f"/module/scripts/utils/run_effective_set_cli.sh --env-id={full_env_name}", + "--envs-path=$CI_PROJECT_DIR/environments", + f"--output=$CI_PROJECT_DIR/environments/{full_env_name}/effective-set" + ] + sd_path = resolve_sd_path() + if sd_path.is_file(): + cmdb_cli_cmd_call.extend([ + "--registries=${CI_PROJECT_DIR}/configuration/registry.yml", + "--sboms-path=$sboms_path", + "--sd-path=$sd_path", + ]) + + effective_set_config = getenv("EFFECTIVE_SET_CONFIG") + effective_set_config_dict = {} + if effective_set_config: + logger.info(f"EFFECTIVE_SET_CONFIG: {effective_set_config}") + effective_set_config_dict = json.loads(effective_set_config) + # validate_topology_context_mode(effective_set_config_dict, full_env_name) + + if effective_set_config: + handle_effective_set_config(effective_set_config) + effective_set_output = openJson('/tmp/effective_set_output.json') + extra_args = " ".join(effective_set_output.get("extra_args") or []) + cmdb_cli_cmd_call.extend([extra_args]) + + deployment_id = getenv("DEPLOYMENT_SESSION_ID") + if deployment_id: + cmdb_cli_cmd_call.append(f"--extra_params=DEPLOYMENT_SESSION_ID={deployment_id}") + + custom_params = getenv("CUSTOM_PARAMS") + if custom_params: + logger.info(f"custom_params : {custom_params}") + cmdb_cli_cmd_call.append(f"--custom-params='{custom_params}'") + + # run java effective set cli + subprocess.run(["sh", cmdb_cli_cmd_call], check=True) + + encrypt_all_cred_files_for_env() + + +def validate_topology_context_mode(effective_set_config_dict, full_env_name): + effective_set_version = effective_set_config_dict.get("version") or "v2.0" + sd = bool(getenv("SD_DATA")) or bool(getenv("SD_VERSION")) + full_sd_path = Path( + f'{getenv('CI_PROJECT_DIR')}/environments/{full_env_name}/Inventory/solution-descriptor/sd.yaml') + # effective set generation in version 1.0 does not support no SBOMs mode + if not (full_sd_path.exists() and sd) and effective_set_version.lower() == "v1.0": + raise ValueError("Feature generation effective set for pipeline and topology context is not supported for v1.0") diff --git a/build_effective_set_generator/scripts/main.py b/build_effective_set_generator/scripts/main.py deleted file mode 100644 index ce2669fa6..000000000 --- a/build_effective_set_generator/scripts/main.py +++ /dev/null @@ -1,26 +0,0 @@ -import click -from envgenehelper import encrypt_all_cred_files_for_env, decrypt_all_cred_files_for_env, validate_creds - - -@click.group(chain=True) -def crypt_manager(): - pass - - -@crypt_manager.command("decrypt_cred_files") -def decrypt_cred_files(): - decrypt_all_cred_files_for_env() - - -@crypt_manager.command("encrypt_cred_files") -def encrypt_cred_files(): - encrypt_all_cred_files_for_env() - - -@crypt_manager.command("validate_creds") -def validate_credentials(): - validate_creds() - - -if __name__ == "__main__": - crypt_manager() diff --git a/build_effective_set_generator/scripts/sboms_retention_policy.py b/build_effective_set_generator/scripts/sboms_retention_policy.py index 2cb1a4447..de19acbc9 100644 --- a/build_effective_set_generator/scripts/sboms_retention_policy.py +++ b/build_effective_set_generator/scripts/sboms_retention_policy.py @@ -28,7 +28,3 @@ def sboms_retention_policy(): cleanup_dir_by_age(app_sbom_dir, sbom_retention.keep_versions_per_app) cleanup_dir_by_size(sboms_dir, CI_JOB_ARTIFACT_MAX_SIZE_MB) - - -if __name__ == "__main__": - sboms_retention_policy() diff --git a/build_pipegene/scripts/effective_set_job.py b/build_pipegene/scripts/effective_set_job.py index ca81f793f..968e1bd1b 100644 --- a/build_pipegene/scripts/effective_set_job.py +++ b/build_pipegene/scripts/effective_set_job.py @@ -10,72 +10,15 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluster_name, params): - logger.info(f'Prepare generate-effective-set job for {full_env_name}') - logger.info(f'Cleanup_targets: {cleanup_targets}') - - app_reg_defs_job = params["APP_REG_DEFS_JOB"] - artifact_app_defs_path = params["APP_DEFS_PATH"] - artifact_reg_defs_path = params["REG_DEFS_PATH"] - deployment_id = params["DEPLOYMENT_SESSION_ID"] - effective_set_config = params["EFFECTIVE_SET_CONFIG"] - if "CUSTOM_PARAMS" in params: - custom_params = params["CUSTOM_PARAMS"] - - is_local_app_def = artifact_app_defs_path and artifact_reg_defs_path and app_reg_defs_job - sd_path = resolve_sd_path() - # TODO it is necessary to remove unnecessary calls, leave only script calls in such jobs! bad for gsf delivery script = [ - # Overriding sd_path to pick the correct value for CI_PROJECT_DIR - f'base_env_path="$CI_PROJECT_DIR/environments/{full_env_name}";', - 'app_defs_path="$base_env_path/AppDefs";', - 'reg_defs_path="$base_env_path/RegDefs";', - 'sboms_path="$CI_PROJECT_DIR/sboms";', - f'sd_path="$base_env_path/Inventory/solution-descriptor/{sd_path.name}";', # cert handling for java 'mkdir -p ${CI_PROJECT_DIR}/configuration/certs/', 'if [ -f /default_cert.pem ]; then cp /default_cert.pem "${CI_PROJECT_DIR}/configuration/certs/"; fi', 'for cert in "${CI_PROJECT_DIR}/configuration/certs/*" ; do [ -f "$cert" ] && keytool -import -trustcacerts -alias "$(basename "$cert")" -file "$cert" -keystore /etc/ssl/certs/keystore.jks -storepass changeit -noprompt; done', - 'python3 /module/scripts/main.py decrypt_cred_files', - f'[ -n "$APP_REG_DEFS_JOB" ] && [ -n "$APP_DEFS_PATH" ] && mkdir -p $app_defs_path && cp -rf {artifact_app_defs_path}/* $app_defs_path', - f'[ -n "$APP_REG_DEFS_JOB" ] && [ -n "$REG_DEFS_PATH" ] && mkdir -p $reg_defs_path && cp -fr {artifact_reg_defs_path}/* $reg_defs_path', - 'python3 /module/scripts/main.py validate_creds', - 'python3 /module/scripts/sboms_retention_policy.py' - ] - cmdb_cli_cmd_call = [ - f"/module/scripts/utils/entrypoint.sh --env-id={full_env_name}", - "--envs-path=$CI_PROJECT_DIR/environments", - f"--output=$CI_PROJECT_DIR/environments/{full_env_name}/effective-set" + 'python /scripts/effective_set_entrypoint.py' ] - effective_set_config_dict = {} - if effective_set_config: - effective_set_config_dict = json.loads(effective_set_config) - validate_topology_context_mode(effective_set_config_dict, full_env_name, params) - if sd_path.is_file(): - cmdb_cli_cmd_call.extend([ - "--registries=${CI_PROJECT_DIR}/configuration/registry.yml", - "--sboms-path=$sboms_path", - "--sd-path=$sd_path", - ]) - - logger.info(f'Prepare generate_effective_set job for {full_env_name}.') - if effective_set_config: - logger.info(f"EFFECTIVE_SET_CONFIG: {effective_set_config}") - script.extend([ - f"python3 /module/scripts/handle_effective_set_config.py --effective-set-config '{effective_set_config}'", - 'extra_args=$(jq -r \'.extra_args // [] | join(" ")\' /tmp/effective_set_output.json)', - ]) - cmdb_cli_cmd_call.extend(["$extra_args"]) - if deployment_id: - cmdb_cli_cmd_call.extend([f"--extra_params=DEPLOYMENT_SESSION_ID={deployment_id}"]) - - if custom_params: - logger.info(f"custom_params : {custom_params}") - cmdb_cli_cmd_call.extend([f"--custom-params='{custom_params}'"]) - script.append(" ".join(cmdb_cli_cmd_call)) - script.append('python3 /module/scripts/main.py encrypt_cred_files') - generate_effective_set_params = { "name": f'generate_effective_set.{full_env_name}', "image": '${effective_set_generator_image}', @@ -113,12 +56,3 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste pipeline.add_children(generate_effective_set_job) return generate_effective_set_job - - -def validate_topology_context_mode(effective_set_config_dict, full_env_name, params): - effective_set_version = effective_set_config_dict.get("version") or "v2.0" - sd = bool(params["SD_DATA"]) or bool(params["SD_VERSION"]) - full_sd_path = Path(f'{getenv('CI_PROJECT_DIR')}/environments/{full_env_name}/Inventory/solution-descriptor/sd.yaml') - # effective set generation in version 1.0 does not support no SBOMs mode - if not (full_sd_path.exists() and sd) and effective_set_version.lower() == "v1.0": - raise ValueError("Feature generation effective set for pipeline and topology context is not supported for v1.0") diff --git a/scripts/utils/entrypoint.sh b/scripts/utils/run_effective_set_cli.sh old mode 100755 new mode 100644 similarity index 100% rename from scripts/utils/entrypoint.sh rename to scripts/utils/run_effective_set_cli.sh From 5e211ac6897db7568189dfb39e1052704360a247 Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Mon, 4 May 2026 17:22:21 +0500 Subject: [PATCH 06/24] feat: refactor effective set job --- .../scripts/effective_set_entrypoint.py | 2 -- build_pipegene/scripts/effective_set_job.py | 13 +++++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/build_effective_set_generator/scripts/effective_set_entrypoint.py b/build_effective_set_generator/scripts/effective_set_entrypoint.py index 158f49015..cc9b2e580 100644 --- a/build_effective_set_generator/scripts/effective_set_entrypoint.py +++ b/build_effective_set_generator/scripts/effective_set_entrypoint.py @@ -44,9 +44,7 @@ def effective_set_entrypoint(): ]) effective_set_config = getenv("EFFECTIVE_SET_CONFIG") - effective_set_config_dict = {} if effective_set_config: - logger.info(f"EFFECTIVE_SET_CONFIG: {effective_set_config}") effective_set_config_dict = json.loads(effective_set_config) # validate_topology_context_mode(effective_set_config_dict, full_env_name) diff --git a/build_pipegene/scripts/effective_set_job.py b/build_pipegene/scripts/effective_set_job.py index 968e1bd1b..51a79ea2a 100644 --- a/build_pipegene/scripts/effective_set_job.py +++ b/build_pipegene/scripts/effective_set_job.py @@ -1,8 +1,7 @@ import json from os import getenv, environ -from pathlib import Path -from envgenehelper import cleanup_targets, resolve_sd_path +from envgenehelper import cleanup_targets from envgenehelper import logger from gcip import WhenStatement, Need @@ -37,6 +36,10 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste } needs = [] + app_reg_defs_job = params["APP_REG_DEFS_JOB"] + artifact_app_defs_path = params["APP_DEFS_PATH"] + artifact_reg_defs_path = params["REG_DEFS_PATH"] + is_local_app_def = artifact_app_defs_path and artifact_reg_defs_path and app_reg_defs_job if is_local_app_def: # gcip library doesn't allow to create a Need object that has the same pipeline as one it runs within. # We need to specify pipeline because generated job will be ran in child pipeline @@ -45,9 +48,15 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste environ['CI_PIPELINE_ID'] = '0000000' needs.append(Need(job=app_reg_defs_job, pipeline=real_ci_pipe_id, artifacts=True)) environ['CI_PIPELINE_ID'] = real_ci_pipe_id + generate_effective_set_job = job_instance(params=generate_effective_set_params, needs=needs, vars=generate_effective_set_vars) + effective_set_config_dict = {} + effective_set_config = params["EFFECTIVE_SET_CONFIG"] + if effective_set_config: + logger.info(f"EFFECTIVE_SET_CONFIG: {effective_set_config}") + effective_set_config_dict = json.loads(effective_set_config) effective_set_expiry = effective_set_config_dict.get("effective_set_expiry") or "1 hour" logger.info(f"effective set expiry value '{effective_set_expiry}'") generate_effective_set_job.artifacts.expire_in = effective_set_expiry From ae033f6311f069568876b00430f667885a3c144c Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Mon, 4 May 2026 17:39:57 +0500 Subject: [PATCH 07/24] feat: refactor effective set job --- .../scripts/effective_set_entrypoint.py | 16 ------------- build_pipegene/scripts/effective_set_job.py | 23 +++++++++++++++---- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/build_effective_set_generator/scripts/effective_set_entrypoint.py b/build_effective_set_generator/scripts/effective_set_entrypoint.py index cc9b2e580..e850dba9e 100644 --- a/build_effective_set_generator/scripts/effective_set_entrypoint.py +++ b/build_effective_set_generator/scripts/effective_set_entrypoint.py @@ -1,7 +1,5 @@ -import json import subprocess from os import getenv -from pathlib import Path from envgenehelper import decrypt_all_cred_files_for_env, copy_path, validate_creds, openJson, \ encrypt_all_cred_files_for_env, resolve_sd_path, logger @@ -44,10 +42,6 @@ def effective_set_entrypoint(): ]) effective_set_config = getenv("EFFECTIVE_SET_CONFIG") - if effective_set_config: - effective_set_config_dict = json.loads(effective_set_config) - # validate_topology_context_mode(effective_set_config_dict, full_env_name) - if effective_set_config: handle_effective_set_config(effective_set_config) effective_set_output = openJson('/tmp/effective_set_output.json') @@ -67,13 +61,3 @@ def effective_set_entrypoint(): subprocess.run(["sh", cmdb_cli_cmd_call], check=True) encrypt_all_cred_files_for_env() - - -def validate_topology_context_mode(effective_set_config_dict, full_env_name): - effective_set_version = effective_set_config_dict.get("version") or "v2.0" - sd = bool(getenv("SD_DATA")) or bool(getenv("SD_VERSION")) - full_sd_path = Path( - f'{getenv('CI_PROJECT_DIR')}/environments/{full_env_name}/Inventory/solution-descriptor/sd.yaml') - # effective set generation in version 1.0 does not support no SBOMs mode - if not (full_sd_path.exists() and sd) and effective_set_version.lower() == "v1.0": - raise ValueError("Feature generation effective set for pipeline and topology context is not supported for v1.0") diff --git a/build_pipegene/scripts/effective_set_job.py b/build_pipegene/scripts/effective_set_job.py index 51a79ea2a..c2ab07f5c 100644 --- a/build_pipegene/scripts/effective_set_job.py +++ b/build_pipegene/scripts/effective_set_job.py @@ -1,5 +1,6 @@ import json from os import getenv, environ +from pathlib import Path from envgenehelper import cleanup_targets from envgenehelper import logger @@ -9,6 +10,14 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluster_name, params): + logger.info(f'Prepare generate-effective-set job for {full_env_name}') + + effective_set_config = params["EFFECTIVE_SET_CONFIG"] + if effective_set_config: + logger.info(f"EFFECTIVE_SET_CONFIG: {effective_set_config}") + effective_set_config_dict = json.loads(effective_set_config) + validate_topology_context_mode(effective_set_config_dict, full_env_name, params) + script = [ # cert handling for java 'mkdir -p ${CI_PROJECT_DIR}/configuration/certs/', @@ -53,10 +62,6 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste vars=generate_effective_set_vars) effective_set_config_dict = {} - effective_set_config = params["EFFECTIVE_SET_CONFIG"] - if effective_set_config: - logger.info(f"EFFECTIVE_SET_CONFIG: {effective_set_config}") - effective_set_config_dict = json.loads(effective_set_config) effective_set_expiry = effective_set_config_dict.get("effective_set_expiry") or "1 hour" logger.info(f"effective set expiry value '{effective_set_expiry}'") generate_effective_set_job.artifacts.expire_in = effective_set_expiry @@ -65,3 +70,13 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste pipeline.add_children(generate_effective_set_job) return generate_effective_set_job + + +def validate_topology_context_mode(effective_set_config_dict, full_env_name, params): + effective_set_version = effective_set_config_dict.get("version") or "v2.0" + sd = bool(params["SD_DATA"]) or bool(params["SD_VERSION"]) + full_sd_path = Path( + f'{getenv('CI_PROJECT_DIR')}/environments/{full_env_name}/Inventory/solution-descriptor/sd.yaml') + # effective set generation in version 1.0 does not support no SBOMs mode + if not (full_sd_path.exists() and sd) and effective_set_version.lower() == "v1.0": + raise ValueError("Feature generation effective set for pipeline and topology context is not supported for v1.0") From 85acc6adde1baa186cedb4743278930b215fb1ac Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Mon, 4 May 2026 18:55:27 +0500 Subject: [PATCH 08/24] feat: add feature toggle --- python/envgene/envgenehelper/sd_helper.py | 29 +++++++++++++---------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/python/envgene/envgenehelper/sd_helper.py b/python/envgene/envgenehelper/sd_helper.py index dd476b74a..e7566a2cd 100644 --- a/python/envgene/envgenehelper/sd_helper.py +++ b/python/envgene/envgenehelper/sd_helper.py @@ -269,15 +269,20 @@ def calculate_merge_mode(sd_merge_mode, sd_delta) -> MergeType: def resolve_sd_path() -> Path: - sd_delta = getenv('SD_DELTA') - sd_merge_mode = getenv("SD_REPO_MERGE_MODE") - base_sd_path = get_sd_dir() - merge_mode = calculate_merge_mode(sd_merge_mode, sd_delta) - full_sd_path = base_sd_path.joinpath(SD_FILE_NAME) - sd_version = getenv("SD_VERSION") - sd_data = getenv("SD_DATA") - sd_input = bool(sd_data) or bool(sd_version) - if merge_mode == MergeType.REPLACE or (not sd_input and full_sd_path.is_file()): - return full_sd_path - elif merge_mode == MergeType.BASIC or merge_mode == MergeType.EXTENDED: - return base_sd_path.joinpath(DELTA_SD_FILE_NAME) + partial_gen = get_envgene_config_yaml().get("effective_set_partial_generation") + sd_dir = get_sd_dir() + full_sd_path = sd_dir.joinpath(SD_FILE_NAME) + if partial_gen: + logger.info("effective_set_partial_generation feature enabled") + sd_delta = getenv('SD_DELTA') + sd_merge_mode = getenv("SD_REPO_MERGE_MODE") + merge_mode = calculate_merge_mode(sd_merge_mode, sd_delta) + sd_version = getenv("SD_VERSION") + sd_data = getenv("SD_DATA") + sd_input = bool(sd_data) or bool(sd_version) + if merge_mode == MergeType.REPLACE or (not sd_input and full_sd_path.is_file()): + return full_sd_path + elif merge_mode == MergeType.BASIC or merge_mode == MergeType.EXTENDED: + return sd_dir.joinpath(DELTA_SD_FILE_NAME) + logger.info("effective_set_partial_generation feature disabled") + return full_sd_path From df08c067593911f126c7d38b89cabc05178bbfbf Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Tue, 5 May 2026 14:04:55 +0500 Subject: [PATCH 09/24] feat: delete EXCLUDE_CLEANUP_TARGETS --- build_pipegene/scripts/effective_set_job.py | 2 -- python/envgene/envgenehelper/sd_helper.py | 6 ++++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build_pipegene/scripts/effective_set_job.py b/build_pipegene/scripts/effective_set_job.py index c2ab07f5c..80e7749c4 100644 --- a/build_pipegene/scripts/effective_set_job.py +++ b/build_pipegene/scripts/effective_set_job.py @@ -2,7 +2,6 @@ from os import getenv, environ from pathlib import Path -from envgenehelper import cleanup_targets from envgenehelper import logger from gcip import WhenStatement, Need @@ -40,7 +39,6 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste "ENV_NAME": env_name, "INSTANCES_DIR": "${CI_PROJECT_DIR}/environments", "effective_set_generator_image": "$effective_set_generator_image", - "EXCLUDE_CLEANUP_TARGETS": " ".join(cleanup_targets), "FULL_ENV_NAME": full_env_name, } diff --git a/python/envgene/envgenehelper/sd_helper.py b/python/envgene/envgenehelper/sd_helper.py index e7566a2cd..b41930ff2 100644 --- a/python/envgene/envgenehelper/sd_helper.py +++ b/python/envgene/envgenehelper/sd_helper.py @@ -269,11 +269,11 @@ def calculate_merge_mode(sd_merge_mode, sd_delta) -> MergeType: def resolve_sd_path() -> Path: - partial_gen = get_envgene_config_yaml().get("effective_set_partial_generation") + partial_gen = get_envgene_config_yaml().get("partial_effective_set_generation") sd_dir = get_sd_dir() full_sd_path = sd_dir.joinpath(SD_FILE_NAME) if partial_gen: - logger.info("effective_set_partial_generation feature enabled") + logger.info("partial effective set generation feature enabled") sd_delta = getenv('SD_DELTA') sd_merge_mode = getenv("SD_REPO_MERGE_MODE") merge_mode = calculate_merge_mode(sd_merge_mode, sd_delta) @@ -284,5 +284,7 @@ def resolve_sd_path() -> Path: return full_sd_path elif merge_mode == MergeType.BASIC or merge_mode == MergeType.EXTENDED: return sd_dir.joinpath(DELTA_SD_FILE_NAME) + elif merge_mode == MergeType.BASIC_EXCLUSION: + return logger.info("effective_set_partial_generation feature disabled") return full_sd_path From be47d7ce5492e5bd0379b077851c8f16bb4ac89b Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Tue, 5 May 2026 17:58:58 +0500 Subject: [PATCH 10/24] feat: add clean up --- .../devops/cli/utils/FileSystemUtils.java | 9 +---- .../scripts/effective_set_entrypoint.py | 33 +++++++++++++++++-- .../envgene/envgenehelper/business_helper.py | 5 --- python/envgene/envgenehelper/constants.py | 9 +++++ python/envgene/envgenehelper/sd_helper.py | 8 ++++- 5 files changed, 48 insertions(+), 16 deletions(-) diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/FileSystemUtils.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/FileSystemUtils.java index 61d70e14c..399adb796 100644 --- a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/FileSystemUtils.java +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/FileSystemUtils.java @@ -68,9 +68,6 @@ public void createEffectiveSetFolder(Optional solutionDescriptor private void createEffectiveSetTwo(Optional solutionDescriptor) throws IOException { File file = getFileFromGivenPath(data.getOutputDir()); - if (file.exists()) { - FileUtils.forceDelete(file); - } file.mkdir(); Path pipelinePath = getFileFromGivenPath(data.getOutputDir(), "pipeline").toPath(); Files.createDirectories(pipelinePath); @@ -92,13 +89,9 @@ private void createEffectiveSetTwo(Optional solutionDescriptor) })); } - private void createEffectiveSetOne(List applicationDTOList) throws IOException { + private void createEffectiveSetOne(List applicationDTOList) { File file = getFileFromGivenPath(data.getOutputDir()); - if (file.exists()) { - FileUtils.forceDelete(file); - } file.mkdir(); - applicationDTOList .forEach(app -> { Path appPath = getFileFromGivenPath(data.getOutputDir(), app.getNamespace(), app.getAppName()).toPath(); diff --git a/build_effective_set_generator/scripts/effective_set_entrypoint.py b/build_effective_set_generator/scripts/effective_set_entrypoint.py index e850dba9e..0cfaf75ff 100644 --- a/build_effective_set_generator/scripts/effective_set_entrypoint.py +++ b/build_effective_set_generator/scripts/effective_set_entrypoint.py @@ -2,7 +2,8 @@ from os import getenv from envgenehelper import decrypt_all_cred_files_for_env, copy_path, validate_creds, openJson, \ - encrypt_all_cred_files_for_env, resolve_sd_path, logger + encrypt_all_cred_files_for_env, resolve_sd_path, logger, get_current_env_dir_from_env_vars, cleanup_dir, \ + get_envgene_config_yaml, openYaml, ESGenerationContext from handle_effective_set_config import handle_effective_set_config from sboms_retention_policy import sboms_retention_policy @@ -28,10 +29,12 @@ def effective_set_entrypoint(): validate_creds() sboms_retention_policy() + effective_set_dir = f"{get_current_env_dir_from_env_vars()}/effective-set" + cmdb_cli_cmd_call = [ f"/module/scripts/utils/run_effective_set_cli.sh --env-id={full_env_name}", "--envs-path=$CI_PROJECT_DIR/environments", - f"--output=$CI_PROJECT_DIR/environments/{full_env_name}/effective-set" + f"--output={effective_set_dir}" ] sd_path = resolve_sd_path() if sd_path.is_file(): @@ -40,6 +43,32 @@ def effective_set_entrypoint(): "--sboms-path=$sboms_path", "--sd-path=$sd_path", ]) + partial_gen = get_envgene_config_yaml().get("partial_effective_set_generation") + if partial_gen: + sd = openYaml(sd_path) + apps = sd.get('applications', []) + deploy_postfixes = { + app.get("deployPostfix") + for app in apps + if app.get("deployPostfix") is not None + } + for ns in deploy_postfixes: + cleanup_dir(effective_set_dir.join(ESGenerationContext.CLEANUP.value).join(ns)) + for app in apps: + app_name = app.get("version").split(':')[0] + deploy_postfix = app.get("deployPostfix") + for_cleanup = (effective_set_dir.join(deploy_postfix).join(ESGenerationContext.RUNTIME.value) + .join(app_name)) + cleanup_dir(for_cleanup) + for_cleanup = (effective_set_dir.join(deploy_postfix).join(ESGenerationContext.DEPLOYMENT.value) + .join(app_name)) + cleanup_dir(for_cleanup) + cleanup_dir(effective_set_dir.join(ESGenerationContext.TOPOLOGY.value)) + cleanup_dir(effective_set_dir.join(ESGenerationContext.PIPELINE.value)) + else: + cleanup_dir(effective_set_dir) + else: + cleanup_dir(effective_set_dir) effective_set_config = getenv("EFFECTIVE_SET_CONFIG") if effective_set_config: diff --git a/python/envgene/envgenehelper/business_helper.py b/python/envgene/envgenehelper/business_helper.py index 4c53bae4a..de01528ac 100644 --- a/python/envgene/envgenehelper/business_helper.py +++ b/python/envgene/envgenehelper/business_helper.py @@ -27,8 +27,6 @@ DEFAULT_PASSPORT_NAME = "passport" DEFAULT_PASSPORT_DIR_NAME = "cloud-passport" INV_GEN_CREDS_PATH = "Inventory/credentials/inventory_generation_creds.yml" -SD_FILE_NAME = "sd.yaml" -DELTA_SD_FILE_NAME = "delta_sd.yaml" TEMPLATE_DIR_PATTERN = re.compile(r'/from_(\w+_)?template/') @@ -460,6 +458,3 @@ def is_from_template_dir(file_path: str) -> bool: def get_sboms_dir(work_dir) -> Path: return Path(work_dir) / "sboms" - -def get_sd_dir() -> Path: - return Path(f'{get_current_env_dir_from_env_vars()}/{INVENTORY_DIR_NAME}/solution-descriptor/') diff --git a/python/envgene/envgenehelper/constants.py b/python/envgene/envgenehelper/constants.py index f7f2f3c10..61ea9a52c 100644 --- a/python/envgene/envgenehelper/constants.py +++ b/python/envgene/envgenehelper/constants.py @@ -1,4 +1,5 @@ # envgenehelper/constants.py +from enum import Enum cleanup_targets = [ "Applications", @@ -11,3 +12,11 @@ ] CI_JOB_ARTIFACT_MAX_SIZE_MB = 1200 # 80% from limit 1.5 + + +class ESGenerationContext(Enum): + TOPOLOGY = "topology" + PIPELINE = "pipeline" + DEPLOYMENT = "deployment" + RUNTIME = "runtime" + CLEANUP = "cleanup" diff --git a/python/envgene/envgenehelper/sd_helper.py b/python/envgene/envgenehelper/sd_helper.py index b41930ff2..f4a7ffc1e 100644 --- a/python/envgene/envgenehelper/sd_helper.py +++ b/python/envgene/envgenehelper/sd_helper.py @@ -3,7 +3,8 @@ MERGE_IMPOSSIBLE = "SD merge error:\nDelta SD contains a new applications, but doesn't contain this application in the deployGraph.\nSD Merge is impossible." NEW_CHUNK_ERROR = "SD merge error:\nDelta SD contains a new chunk\nSD Merge is impossible." NO_DEPLOY_GRAPH_ERROR = "SD merge error:\nDelta SD contains deployGraph, but Full SD doesn't contain deployGraph.\nSD Merge is impossible." - +SD_FILE_NAME = "sd.yaml" +DELTA_SD_FILE_NAME = "delta_sd.yaml" def get_app_name(name: str): return name[0:name.find(":")] @@ -268,6 +269,10 @@ def calculate_merge_mode(sd_merge_mode, sd_delta) -> MergeType: return effective_merge_mode +def get_sd_dir() -> Path: + return Path(f'{get_current_env_dir_from_env_vars()}/{INVENTORY_DIR_NAME}/solution-descriptor/') + + def resolve_sd_path() -> Path: partial_gen = get_envgene_config_yaml().get("partial_effective_set_generation") sd_dir = get_sd_dir() @@ -284,6 +289,7 @@ def resolve_sd_path() -> Path: return full_sd_path elif merge_mode == MergeType.BASIC or merge_mode == MergeType.EXTENDED: return sd_dir.joinpath(DELTA_SD_FILE_NAME) + #TODO elif merge_mode == MergeType.BASIC_EXCLUSION: return logger.info("effective_set_partial_generation feature disabled") From 0be0e4914f1bfce89f4fbbfe30cb1af94b5128d4 Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Tue, 5 May 2026 18:19:45 +0500 Subject: [PATCH 11/24] feat: add clean up --- .../scripts/effective_set_entrypoint.py | 76 +++++++++++-------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/build_effective_set_generator/scripts/effective_set_entrypoint.py b/build_effective_set_generator/scripts/effective_set_entrypoint.py index 0cfaf75ff..4f9293304 100644 --- a/build_effective_set_generator/scripts/effective_set_entrypoint.py +++ b/build_effective_set_generator/scripts/effective_set_entrypoint.py @@ -1,9 +1,21 @@ +import os import subprocess from os import getenv -from envgenehelper import decrypt_all_cred_files_for_env, copy_path, validate_creds, openJson, \ - encrypt_all_cred_files_for_env, resolve_sd_path, logger, get_current_env_dir_from_env_vars, cleanup_dir, \ - get_envgene_config_yaml, openYaml, ESGenerationContext +from envgenehelper import ( + decrypt_all_cred_files_for_env, + copy_path, + validate_creds, + openJson, + encrypt_all_cred_files_for_env, + resolve_sd_path, + logger, + get_current_env_dir_from_env_vars, + cleanup_dir, + get_envgene_config_yaml, + openYaml, + ESGenerationContext, +) from handle_effective_set_config import handle_effective_set_config from sboms_retention_policy import sboms_retention_policy @@ -12,30 +24,31 @@ def effective_set_entrypoint(): full_env_name = getenv("FULL_ENV_NAME") ci_project_dir = getenv("CI_PROJECT_DIR") - work_dir = ci_project_dir / "environments" / full_env_name + work_dir = os.path.join(ci_project_dir, "environments", full_env_name) decrypt_all_cred_files_for_env() artifact_app_defs_path = getenv("APP_DEFS_PATH") artifact_reg_defs_path = getenv("REG_DEFS_PATH") app_reg_defs_job = getenv("APP_REG_DEFS_JOB") - is_local_app_def = artifact_app_defs_path and artifact_reg_defs_path and app_reg_defs_job - if is_local_app_def: - app_defs_path = work_dir / "AppDefs" - reg_defs_path = work_dir / "RegDefs" - copy_path(artifact_app_defs_path, app_defs_path) - copy_path(artifact_reg_defs_path, reg_defs_path) + + # local app reg defs + if artifact_app_defs_path and artifact_reg_defs_path and app_reg_defs_job: + copy_path(artifact_app_defs_path, os.path.join(work_dir, "AppDefs")) + copy_path(artifact_reg_defs_path, os.path.join(work_dir, "RegDefs")) validate_creds() sboms_retention_policy() - effective_set_dir = f"{get_current_env_dir_from_env_vars()}/effective-set" + effective_set_dir = get_current_env_dir_from_env_vars() / "effective-set" cmdb_cli_cmd_call = [ - f"/module/scripts/utils/run_effective_set_cli.sh --env-id={full_env_name}", + "/module/scripts/utils/run_effective_set_cli.sh", + f"--env-id={full_env_name}", "--envs-path=$CI_PROJECT_DIR/environments", - f"--output={effective_set_dir}" + f"--output={effective_set_dir}", ] + sd_path = resolve_sd_path() if sd_path.is_file(): cmdb_cli_cmd_call.extend([ @@ -46,36 +59,39 @@ def effective_set_entrypoint(): partial_gen = get_envgene_config_yaml().get("partial_effective_set_generation") if partial_gen: sd = openYaml(sd_path) - apps = sd.get('applications', []) + apps = sd.get("applications", []) + deploy_postfixes = { app.get("deployPostfix") for app in apps - if app.get("deployPostfix") is not None + if app.get("deployPostfix") } + # cleanup per namespace for ns in deploy_postfixes: - cleanup_dir(effective_set_dir.join(ESGenerationContext.CLEANUP.value).join(ns)) + cleanup_dir(effective_set_dir / ESGenerationContext.CLEANUP.value / ns) + + # cleanup per app for app in apps: - app_name = app.get("version").split(':')[0] + app_name = app.get("version").split(":")[0] deploy_postfix = app.get("deployPostfix") - for_cleanup = (effective_set_dir.join(deploy_postfix).join(ESGenerationContext.RUNTIME.value) - .join(app_name)) - cleanup_dir(for_cleanup) - for_cleanup = (effective_set_dir.join(deploy_postfix).join(ESGenerationContext.DEPLOYMENT.value) - .join(app_name)) - cleanup_dir(for_cleanup) - cleanup_dir(effective_set_dir.join(ESGenerationContext.TOPOLOGY.value)) - cleanup_dir(effective_set_dir.join(ESGenerationContext.PIPELINE.value)) + cleanup_dir(effective_set_dir / deploy_postfix / ESGenerationContext.RUNTIME.value / app_name) + cleanup_dir(effective_set_dir / deploy_postfix / ESGenerationContext.DEPLOYMENT.value / app_name) + + cleanup_dir(effective_set_dir / ESGenerationContext.TOPOLOGY.value) + cleanup_dir(effective_set_dir / ESGenerationContext.PIPELINE.value) + else: cleanup_dir(effective_set_dir) else: + # for pipeline and topology context generation in no sbom mode cleanup_dir(effective_set_dir) effective_set_config = getenv("EFFECTIVE_SET_CONFIG") if effective_set_config: handle_effective_set_config(effective_set_config) - effective_set_output = openJson('/tmp/effective_set_output.json') - extra_args = " ".join(effective_set_output.get("extra_args") or []) - cmdb_cli_cmd_call.extend([extra_args]) + effective_set_output = openJson("/tmp/effective_set_output.json") + extra_args = effective_set_output.get("extra_args") or [] + cmdb_cli_cmd_call.extend(extra_args) deployment_id = getenv("DEPLOYMENT_SESSION_ID") if deployment_id: @@ -83,8 +99,8 @@ def effective_set_entrypoint(): custom_params = getenv("CUSTOM_PARAMS") if custom_params: - logger.info(f"custom_params : {custom_params}") - cmdb_cli_cmd_call.append(f"--custom-params='{custom_params}'") + logger.info(f"custom_params: {custom_params}") + cmdb_cli_cmd_call.append(f"--custom-params={custom_params}") # run java effective set cli subprocess.run(["sh", cmdb_cli_cmd_call], check=True) From 4b059a3fd1b08685dc4c54c1f1c2a72f4a8dbcd2 Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Tue, 5 May 2026 19:13:27 +0500 Subject: [PATCH 12/24] feat: add merge mapping --- .../scripts/effective_set_entrypoint.py | 11 ++++++++--- build_pipegene/scripts/effective_set_job.py | 2 +- python/envgene/envgenehelper/constants.py | 3 +++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/build_effective_set_generator/scripts/effective_set_entrypoint.py b/build_effective_set_generator/scripts/effective_set_entrypoint.py index 4f9293304..7f5b58211 100644 --- a/build_effective_set_generator/scripts/effective_set_entrypoint.py +++ b/build_effective_set_generator/scripts/effective_set_entrypoint.py @@ -15,6 +15,7 @@ get_envgene_config_yaml, openYaml, ESGenerationContext, + ES_MAPPING_FILE ) from handle_effective_set_config import handle_effective_set_config @@ -74,8 +75,8 @@ def effective_set_entrypoint(): for app in apps: app_name = app.get("version").split(":")[0] deploy_postfix = app.get("deployPostfix") - cleanup_dir(effective_set_dir / deploy_postfix / ESGenerationContext.RUNTIME.value / app_name) - cleanup_dir(effective_set_dir / deploy_postfix / ESGenerationContext.DEPLOYMENT.value / app_name) + cleanup_dir(effective_set_dir / ESGenerationContext.RUNTIME.value / deploy_postfix / app_name) + cleanup_dir(effective_set_dir / ESGenerationContext.DEPLOYMENT.value / deploy_postfix / app_name) cleanup_dir(effective_set_dir / ESGenerationContext.TOPOLOGY.value) cleanup_dir(effective_set_dir / ESGenerationContext.PIPELINE.value) @@ -83,7 +84,7 @@ def effective_set_entrypoint(): else: cleanup_dir(effective_set_dir) else: - # for pipeline and topology context generation in no sbom mode + # for pipeline and topology context generation in no sd mode cleanup_dir(effective_set_dir) effective_set_config = getenv("EFFECTIVE_SET_CONFIG") @@ -102,6 +103,10 @@ def effective_set_entrypoint(): logger.info(f"custom_params: {custom_params}") cmdb_cli_cmd_call.append(f"--custom-params={custom_params}") + cleanup_mapping = effective_set_dir / ESGenerationContext.CLEANUP.value / ES_MAPPING_FILE + runtime_mapping = effective_set_dir / ESGenerationContext.RUNTIME.value / ES_MAPPING_FILE + deployment_mapping = effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ES_MAPPING_FILE + # run java effective set cli subprocess.run(["sh", cmdb_cli_cmd_call], check=True) diff --git a/build_pipegene/scripts/effective_set_job.py b/build_pipegene/scripts/effective_set_job.py index 80e7749c4..6693ce4c1 100644 --- a/build_pipegene/scripts/effective_set_job.py +++ b/build_pipegene/scripts/effective_set_job.py @@ -75,6 +75,6 @@ def validate_topology_context_mode(effective_set_config_dict, full_env_name, par sd = bool(params["SD_DATA"]) or bool(params["SD_VERSION"]) full_sd_path = Path( f'{getenv('CI_PROJECT_DIR')}/environments/{full_env_name}/Inventory/solution-descriptor/sd.yaml') - # effective set generation in version 1.0 does not support no SBOMs mode + # effective set generation in version 1.0 does not support no sd mode if not (full_sd_path.exists() and sd) and effective_set_version.lower() == "v1.0": raise ValueError("Feature generation effective set for pipeline and topology context is not supported for v1.0") diff --git a/python/envgene/envgenehelper/constants.py b/python/envgene/envgenehelper/constants.py index 61ea9a52c..5699b8c3b 100644 --- a/python/envgene/envgenehelper/constants.py +++ b/python/envgene/envgenehelper/constants.py @@ -20,3 +20,6 @@ class ESGenerationContext(Enum): DEPLOYMENT = "deployment" RUNTIME = "runtime" CLEANUP = "cleanup" + + +ES_MAPPING_FILE = "mapping.yaml" From f9f429edfd0982264e2a7ec1f08d6c2a2639d327 Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Tue, 5 May 2026 19:27:48 +0500 Subject: [PATCH 13/24] feat: add merge mapping --- .../scripts/effective_set_entrypoint.py | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/build_effective_set_generator/scripts/effective_set_entrypoint.py b/build_effective_set_generator/scripts/effective_set_entrypoint.py index 7f5b58211..2051047e5 100644 --- a/build_effective_set_generator/scripts/effective_set_entrypoint.py +++ b/build_effective_set_generator/scripts/effective_set_entrypoint.py @@ -1,6 +1,7 @@ import os import subprocess from os import getenv +from pathlib import Path from envgenehelper import ( decrypt_all_cred_files_for_env, @@ -15,7 +16,7 @@ get_envgene_config_yaml, openYaml, ESGenerationContext, - ES_MAPPING_FILE + ES_MAPPING_FILE, writeYamlToFile ) from handle_effective_set_config import handle_effective_set_config @@ -103,11 +104,27 @@ def effective_set_entrypoint(): logger.info(f"custom_params: {custom_params}") cmdb_cli_cmd_call.append(f"--custom-params={custom_params}") - cleanup_mapping = effective_set_dir / ESGenerationContext.CLEANUP.value / ES_MAPPING_FILE - runtime_mapping = effective_set_dir / ESGenerationContext.RUNTIME.value / ES_MAPPING_FILE - deployment_mapping = effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ES_MAPPING_FILE + cleanup_mapping_path = effective_set_dir / ESGenerationContext.CLEANUP.value / ES_MAPPING_FILE + runtime_mapping_path = effective_set_dir / ESGenerationContext.RUNTIME.value / ES_MAPPING_FILE + deployment_mapping_path = effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ES_MAPPING_FILE + + cleanup_mapping = openYaml(cleanup_mapping_path) + runtime_mapping = openYaml(runtime_mapping_path) + deployment_mapping = openYaml(deployment_mapping_path) # run java effective set cli subprocess.run(["sh", cmdb_cli_cmd_call], check=True) + new_cleanup_mapping = openYaml(cleanup_mapping_path) + new_runtime_mapping = openYaml(runtime_mapping_path) + new_deployment_mapping = openYaml(deployment_mapping_path) + + cleanup_mapping.update(new_cleanup_mapping) + runtime_mapping.update(new_runtime_mapping) + deployment_mapping.update(new_deployment_mapping) + + writeYamlToFile(cleanup_mapping_path, cleanup_mapping) + writeYamlToFile(runtime_mapping_path, runtime_mapping) + writeYamlToFile(deployment_mapping_path, deployment_mapping) + encrypt_all_cred_files_for_env() From 57dbf5148688d6ba130bde345e7b8a069c9cf86d Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Tue, 5 May 2026 21:00:34 +0500 Subject: [PATCH 14/24] feat: add merge mapping --- .../scripts/effective_set_entrypoint.py | 90 ++++++++++--------- python/envgene/envgenehelper/sd_helper.py | 17 +--- 2 files changed, 50 insertions(+), 57 deletions(-) diff --git a/build_effective_set_generator/scripts/effective_set_entrypoint.py b/build_effective_set_generator/scripts/effective_set_entrypoint.py index 2051047e5..8509602b7 100644 --- a/build_effective_set_generator/scripts/effective_set_entrypoint.py +++ b/build_effective_set_generator/scripts/effective_set_entrypoint.py @@ -1,7 +1,6 @@ import os import subprocess from os import getenv -from pathlib import Path from envgenehelper import ( decrypt_all_cred_files_for_env, @@ -16,7 +15,8 @@ get_envgene_config_yaml, openYaml, ESGenerationContext, - ES_MAPPING_FILE, writeYamlToFile + ES_MAPPING_FILE, + writeYamlToFile ) from handle_effective_set_config import handle_effective_set_config @@ -58,34 +58,34 @@ def effective_set_entrypoint(): "--sboms-path=$sboms_path", "--sd-path=$sd_path", ]) - partial_gen = get_envgene_config_yaml().get("partial_effective_set_generation") - if partial_gen: - sd = openYaml(sd_path) - apps = sd.get("applications", []) - - deploy_postfixes = { - app.get("deployPostfix") - for app in apps - if app.get("deployPostfix") - } - # cleanup per namespace - for ns in deploy_postfixes: - cleanup_dir(effective_set_dir / ESGenerationContext.CLEANUP.value / ns) - - # cleanup per app - for app in apps: - app_name = app.get("version").split(":")[0] - deploy_postfix = app.get("deployPostfix") - cleanup_dir(effective_set_dir / ESGenerationContext.RUNTIME.value / deploy_postfix / app_name) - cleanup_dir(effective_set_dir / ESGenerationContext.DEPLOYMENT.value / deploy_postfix / app_name) - - cleanup_dir(effective_set_dir / ESGenerationContext.TOPOLOGY.value) - cleanup_dir(effective_set_dir / ESGenerationContext.PIPELINE.value) - - else: - cleanup_dir(effective_set_dir) + is_es_partial = get_envgene_config_yaml().get("partial_effective_set_generation") + if is_es_partial and not sd_path.is_file(): + raise ValueError(f"No delta sd or full sd by resolved path {sd_path}") + + if is_es_partial: + sd = openYaml(sd_path) + apps = sd.get("applications", []) + + deploy_postfixes = { + app.get("deployPostfix") + for app in apps + if app.get("deployPostfix") + } + # cleanup per namespace + for ns in deploy_postfixes: + cleanup_dir(effective_set_dir / ESGenerationContext.CLEANUP.value / ns) + + # cleanup per app + for app in apps: + app_name = app.get("version").split(":")[0] + deploy_postfix = app.get("deployPostfix") + cleanup_dir(effective_set_dir / ESGenerationContext.RUNTIME.value / deploy_postfix / app_name) + cleanup_dir(effective_set_dir / ESGenerationContext.DEPLOYMENT.value / deploy_postfix / app_name) + + cleanup_dir(effective_set_dir / ESGenerationContext.TOPOLOGY.value) + cleanup_dir(effective_set_dir / ESGenerationContext.PIPELINE.value) + else: - # for pipeline and topology context generation in no sd mode cleanup_dir(effective_set_dir) effective_set_config = getenv("EFFECTIVE_SET_CONFIG") @@ -104,27 +104,29 @@ def effective_set_entrypoint(): logger.info(f"custom_params: {custom_params}") cmdb_cli_cmd_call.append(f"--custom-params={custom_params}") - cleanup_mapping_path = effective_set_dir / ESGenerationContext.CLEANUP.value / ES_MAPPING_FILE - runtime_mapping_path = effective_set_dir / ESGenerationContext.RUNTIME.value / ES_MAPPING_FILE - deployment_mapping_path = effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ES_MAPPING_FILE + if is_es_partial: + cleanup_mapping_path = effective_set_dir / ESGenerationContext.CLEANUP.value / ES_MAPPING_FILE + runtime_mapping_path = effective_set_dir / ESGenerationContext.RUNTIME.value / ES_MAPPING_FILE + deployment_mapping_path = effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ES_MAPPING_FILE - cleanup_mapping = openYaml(cleanup_mapping_path) - runtime_mapping = openYaml(runtime_mapping_path) - deployment_mapping = openYaml(deployment_mapping_path) + cleanup_mapping = openYaml(cleanup_mapping_path) + runtime_mapping = openYaml(runtime_mapping_path) + deployment_mapping = openYaml(deployment_mapping_path) # run java effective set cli subprocess.run(["sh", cmdb_cli_cmd_call], check=True) - new_cleanup_mapping = openYaml(cleanup_mapping_path) - new_runtime_mapping = openYaml(runtime_mapping_path) - new_deployment_mapping = openYaml(deployment_mapping_path) + if is_es_partial: + new_cleanup_mapping = openYaml(cleanup_mapping_path) + new_runtime_mapping = openYaml(runtime_mapping_path) + new_deployment_mapping = openYaml(deployment_mapping_path) - cleanup_mapping.update(new_cleanup_mapping) - runtime_mapping.update(new_runtime_mapping) - deployment_mapping.update(new_deployment_mapping) + cleanup_mapping.update(new_cleanup_mapping) + runtime_mapping.update(new_runtime_mapping) + deployment_mapping.update(new_deployment_mapping) - writeYamlToFile(cleanup_mapping_path, cleanup_mapping) - writeYamlToFile(runtime_mapping_path, runtime_mapping) - writeYamlToFile(deployment_mapping_path, deployment_mapping) + writeYamlToFile(cleanup_mapping_path, cleanup_mapping) + writeYamlToFile(runtime_mapping_path, runtime_mapping) + writeYamlToFile(deployment_mapping_path, deployment_mapping) encrypt_all_cred_files_for_env() diff --git a/python/envgene/envgenehelper/sd_helper.py b/python/envgene/envgenehelper/sd_helper.py index f4a7ffc1e..55219d2fb 100644 --- a/python/envgene/envgenehelper/sd_helper.py +++ b/python/envgene/envgenehelper/sd_helper.py @@ -276,21 +276,12 @@ def get_sd_dir() -> Path: def resolve_sd_path() -> Path: partial_gen = get_envgene_config_yaml().get("partial_effective_set_generation") sd_dir = get_sd_dir() - full_sd_path = sd_dir.joinpath(SD_FILE_NAME) if partial_gen: - logger.info("partial effective set generation feature enabled") - sd_delta = getenv('SD_DELTA') - sd_merge_mode = getenv("SD_REPO_MERGE_MODE") - merge_mode = calculate_merge_mode(sd_merge_mode, sd_delta) + logger.info("Partial effective set generation feature enabled") + merge_mode = calculate_merge_mode(getenv("SD_REPO_MERGE_MODE"), getenv('SD_DELTA')) sd_version = getenv("SD_VERSION") sd_data = getenv("SD_DATA") sd_input = bool(sd_data) or bool(sd_version) - if merge_mode == MergeType.REPLACE or (not sd_input and full_sd_path.is_file()): - return full_sd_path - elif merge_mode == MergeType.BASIC or merge_mode == MergeType.EXTENDED: + if sd_input and merge_mode in [MergeType.BASIC, MergeType.EXTENDED, MergeType.BASIC_EXCLUSION]: return sd_dir.joinpath(DELTA_SD_FILE_NAME) - #TODO - elif merge_mode == MergeType.BASIC_EXCLUSION: - return - logger.info("effective_set_partial_generation feature disabled") - return full_sd_path + return sd_dir.joinpath(SD_FILE_NAME) From 51a7cfe6dec978c1163afe09c4621d67e7aab4e3 Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Tue, 5 May 2026 21:14:31 +0500 Subject: [PATCH 15/24] feat: fix basic exclude mode --- .../scripts/effective_set_entrypoint.py | 19 +++++-------------- python/envgene/envgenehelper/sd_helper.py | 17 +++++++---------- 2 files changed, 12 insertions(+), 24 deletions(-) diff --git a/build_effective_set_generator/scripts/effective_set_entrypoint.py b/build_effective_set_generator/scripts/effective_set_entrypoint.py index 8509602b7..cf9971567 100644 --- a/build_effective_set_generator/scripts/effective_set_entrypoint.py +++ b/build_effective_set_generator/scripts/effective_set_entrypoint.py @@ -3,20 +3,11 @@ from os import getenv from envgenehelper import ( - decrypt_all_cred_files_for_env, - copy_path, - validate_creds, - openJson, - encrypt_all_cred_files_for_env, - resolve_sd_path, - logger, - get_current_env_dir_from_env_vars, - cleanup_dir, - get_envgene_config_yaml, - openYaml, - ESGenerationContext, - ES_MAPPING_FILE, - writeYamlToFile + decrypt_all_cred_files_for_env, copy_path, validate_creds, + openJson, encrypt_all_cred_files_for_env, resolve_sd_path, + logger, get_current_env_dir_from_env_vars, cleanup_dir, + get_envgene_config_yaml, openYaml, ESGenerationContext, + ES_MAPPING_FILE, writeYamlToFile ) from handle_effective_set_config import handle_effective_set_config diff --git a/python/envgene/envgenehelper/sd_helper.py b/python/envgene/envgenehelper/sd_helper.py index 55219d2fb..03bd2aab1 100644 --- a/python/envgene/envgenehelper/sd_helper.py +++ b/python/envgene/envgenehelper/sd_helper.py @@ -189,10 +189,9 @@ def basic_merge(full_sd, delta_sd): def basic_exclusion_merge(full_sd, delta_sd): """ Merge Delta SD into Full SD using `basic-exclusion-merge` rules: - 1. Matching App => update version from Delta + 1. Matching App, New App => WARN 2. Duplicating App => remove from Full SD - 3. New App => log warning - 4. Output contains only `applications` key + 3. Output contains only `applications` key """ logger.info("Inside basic_exclusion_merge") logger.info(f"Full SD: {full_sd}") @@ -201,7 +200,6 @@ def basic_exclusion_merge(full_sd, delta_sd): delta_apps = delta_sd.get("applications", []) result_apps = [] - # Track matched delta apps matched_delta_indices = set() # Stage 1: Process full SD @@ -209,24 +207,23 @@ def basic_exclusion_merge(full_sd, delta_sd): keep = True for i, d_app in enumerate(delta_apps): if is_duplicating(f_app, d_app): - # Rule 3: Remove duplicating + # Rule 2: Remove duplicating keep = False matched_delta_indices.add(i) break elif is_matching(f_app, d_app): - # Rule 1: Replace matching - result_apps.append(d_app) - keep = False + # Rule 1: Warn about matching apps + logger.warning(f"Warning: Update application '{get_app_name_sd(d_app)}' ignored (matching in Full SD)") matched_delta_indices.add(i) break if keep: result_apps.append(f_app) - # Stage 2: Warn about new apps + # Rule 1: Warn about new apps for i, d_app in enumerate(delta_apps): if i not in matched_delta_indices: # Rule 2: New Application, rejects - logger.info(f"Warning: New application '{get_app_name_sd(d_app)}' ignored (not present in Full SD)") + logger.warning(f"Warning: New application '{get_app_name_sd(d_app)}' ignored (not present in Full SD)") return {"applications": result_apps} From 61775030a4c0c3f68feb3ca94dce9dec5df7baa9 Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Wed, 6 May 2026 15:23:53 +0500 Subject: [PATCH 16/24] feat: fix basic exclude mode --- .../scripts/effective_set_entrypoint.py | 207 ++++++++++++------ python/envgene/envgenehelper/sd_helper.py | 19 +- 2 files changed, 150 insertions(+), 76 deletions(-) diff --git a/build_effective_set_generator/scripts/effective_set_entrypoint.py b/build_effective_set_generator/scripts/effective_set_entrypoint.py index cf9971567..bf8a2a328 100644 --- a/build_effective_set_generator/scripts/effective_set_entrypoint.py +++ b/build_effective_set_generator/scripts/effective_set_entrypoint.py @@ -1,23 +1,46 @@ import os import subprocess from os import getenv +from pathlib import Path from envgenehelper import ( decrypt_all_cred_files_for_env, copy_path, validate_creds, openJson, encrypt_all_cred_files_for_env, resolve_sd_path, logger, get_current_env_dir_from_env_vars, cleanup_dir, get_envgene_config_yaml, openYaml, ESGenerationContext, - ES_MAPPING_FILE, writeYamlToFile + ES_MAPPING_FILE, writeYamlToFile, calculate_merge_mode, delete_dir, get_environment_name_from_full_name ) from handle_effective_set_config import handle_effective_set_config +from python.envgene.envgenehelper import get_sd_dir, SD_FILE_NAME, MergeType from sboms_retention_policy import sboms_retention_policy def effective_set_entrypoint(): + ctx = _prepare_context() + + if ctx["is_partial"]: + delta_sd_path = ctx["delta_sd_path"] + sd_path = ctx["sd_path"].exists() + if not ctx["delta_sd_path"].exists() and ctx["sd_path"].exists(): + raise ValueError( + f"[Partial effective set generation] impossible without delta sd {delta_sd_path} and full sd {sd_path}") + + elif ctx["is_reverse"]: + _run_reverse_merge(ctx) + else: + _run_forward_merge(ctx) + else: + _run_full_generation(ctx) + + encrypt_all_cred_files_for_env() + + +def _prepare_context(): full_env_name = getenv("FULL_ENV_NAME") ci_project_dir = getenv("CI_PROJECT_DIR") work_dir = os.path.join(ci_project_dir, "environments", full_env_name) + merge_mode = calculate_merge_mode(getenv("SD_REPO_MERGE_MODE"), getenv("SD_DELTA")) decrypt_all_cred_files_for_env() @@ -25,7 +48,6 @@ def effective_set_entrypoint(): artifact_reg_defs_path = getenv("REG_DEFS_PATH") app_reg_defs_job = getenv("APP_REG_DEFS_JOB") - # local app reg defs if artifact_app_defs_path and artifact_reg_defs_path and app_reg_defs_job: copy_path(artifact_app_defs_path, os.path.join(work_dir, "AppDefs")) copy_path(artifact_reg_defs_path, os.path.join(work_dir, "RegDefs")) @@ -34,90 +56,145 @@ def effective_set_entrypoint(): sboms_retention_policy() effective_set_dir = get_current_env_dir_from_env_vars() / "effective-set" + sd_path = resolve_sd_path() + + is_partial = get_envgene_config_yaml().get( + "partial_effective_set_generation") and merge_mode.name != MergeType.REPLACE + is_reverse = is_partial and merge_mode.name == MergeType.BASIC_EXCLUSION + + return { + "full_env_name": full_env_name, + "effective_set_dir": effective_set_dir, + "sd_path": sd_path, + "delta_sd_path": delta_sd_path + "is_partial": is_partial, + "is_reverse": is_reverse, + } + + +def _run_full_generation(ctx): + effective_set_dir = ctx["effective_set_dir"] + cmd = _build_cli_cmd(ctx) + cleanup_dir(effective_set_dir) + subprocess.run(["sh", cmd], check=True) + + +def _run_forward_merge(ctx): + effective_set_dir = ctx["effective_set_dir"] + delta_sd_path = ctx["delta_sd_path"] + + cmd = _build_cli_cmd(ctx) + + delta_sd = openYaml(delta_sd_path) + apps = delta_sd.get("applications", []) + + deploy_postfixes = { + app.get("deployPostfix") + for app in apps + if app.get("deployPostfix") + } + + for ns in deploy_postfixes: + cleanup_dir(effective_set_dir / ESGenerationContext.CLEANUP.value / ns) + + for app in apps: + app_name = app.get("version", "").split(":")[0] + ns = app.get("deployPostfix") + cleanup_dir(effective_set_dir / ESGenerationContext.RUNTIME.value / ns / app_name) + cleanup_dir(effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ns / app_name) + + cleanup_dir(effective_set_dir / ESGenerationContext.TOPOLOGY.value) + cleanup_dir(effective_set_dir / ESGenerationContext.PIPELINE.value) + + cleanup_mapping_path = effective_set_dir / ESGenerationContext.CLEANUP.value / ES_MAPPING_FILE + runtime_mapping_path = effective_set_dir / ESGenerationContext.RUNTIME.value / ES_MAPPING_FILE + deployment_mapping_path = effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ES_MAPPING_FILE + + cleanup_mapping = openYaml(cleanup_mapping_path) + runtime_mapping = openYaml(runtime_mapping_path) + deployment_mapping = openYaml(deployment_mapping_path) + + subprocess.run(["sh", cmd], check=True) + + new_cleanup_mapping = openYaml(cleanup_mapping_path) + new_runtime_mapping = openYaml(runtime_mapping_path) + new_deployment_mapping = openYaml(deployment_mapping_path) - cmdb_cli_cmd_call = [ + cleanup_mapping.update(new_cleanup_mapping) + runtime_mapping.update(new_runtime_mapping) + deployment_mapping.update(new_deployment_mapping) + + writeYamlToFile(cleanup_mapping_path, cleanup_mapping) + writeYamlToFile(runtime_mapping_path, runtime_mapping) + writeYamlToFile(deployment_mapping_path, deployment_mapping) + + +def _run_reverse_merge(ctx): + effective_set_dir = ctx["effective_set_dir"] + delta_sd_path = ctx["delta_sd_path"] + + apps = openYaml(delta_sd_path).get("applications", []) + + for app in apps: + app_name = app.get("version", "").split(":")[0] + dp = app.get("deployPostfix") + + runtime_dp = effective_set_dir / ESGenerationContext.RUNTIME.value / dp + deployment_dp = effective_set_dir / ESGenerationContext.DEPLOYMENT.value / dp + cleanup_dp = effective_set_dir / ESGenerationContext.CLEANUP.value / dp + + cleanup_dir(runtime_dp / app_name) + cleanup_dir(deployment_dp / app_name) + + if not any(runtime_dp.iterdir()): + delete_dir(runtime_dp) + delete_dir(cleanup_dp) + delete_dir(deployment_dp) + + ns = get_environment_name_from_full_name(ctx["full_env_name"]) + mapping_paths = [ + effective_set_dir / ESGenerationContext.CLEANUP.value / ES_MAPPING_FILE, + effective_set_dir / ESGenerationContext.RUNTIME.value / ES_MAPPING_FILE, + effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ES_MAPPING_FILE, + ] + for path in mapping_paths: + mapping = openYaml(path, allow_default=True) + if ns in mapping: + mapping.pop(ns) + writeYamlToFile(path, mapping) + + +def _build_cli_cmd(ctx): + effective_set_dir = ctx["effective_set_dir"] + sd_path = ctx["sd_path"] + + cmd = [ "/module/scripts/utils/run_effective_set_cli.sh", - f"--env-id={full_env_name}", + f"--env-id={ctx['full_env_name']}", "--envs-path=$CI_PROJECT_DIR/environments", f"--output={effective_set_dir}", ] - sd_path = resolve_sd_path() if sd_path.is_file(): - cmdb_cli_cmd_call.extend([ + cmd.extend([ "--registries=${CI_PROJECT_DIR}/configuration/registry.yml", "--sboms-path=$sboms_path", "--sd-path=$sd_path", ]) - is_es_partial = get_envgene_config_yaml().get("partial_effective_set_generation") - if is_es_partial and not sd_path.is_file(): - raise ValueError(f"No delta sd or full sd by resolved path {sd_path}") - - if is_es_partial: - sd = openYaml(sd_path) - apps = sd.get("applications", []) - - deploy_postfixes = { - app.get("deployPostfix") - for app in apps - if app.get("deployPostfix") - } - # cleanup per namespace - for ns in deploy_postfixes: - cleanup_dir(effective_set_dir / ESGenerationContext.CLEANUP.value / ns) - - # cleanup per app - for app in apps: - app_name = app.get("version").split(":")[0] - deploy_postfix = app.get("deployPostfix") - cleanup_dir(effective_set_dir / ESGenerationContext.RUNTIME.value / deploy_postfix / app_name) - cleanup_dir(effective_set_dir / ESGenerationContext.DEPLOYMENT.value / deploy_postfix / app_name) - - cleanup_dir(effective_set_dir / ESGenerationContext.TOPOLOGY.value) - cleanup_dir(effective_set_dir / ESGenerationContext.PIPELINE.value) - - else: - cleanup_dir(effective_set_dir) effective_set_config = getenv("EFFECTIVE_SET_CONFIG") if effective_set_config: handle_effective_set_config(effective_set_config) effective_set_output = openJson("/tmp/effective_set_output.json") extra_args = effective_set_output.get("extra_args") or [] - cmdb_cli_cmd_call.extend(extra_args) + cmd.extend(extra_args) deployment_id = getenv("DEPLOYMENT_SESSION_ID") if deployment_id: - cmdb_cli_cmd_call.append(f"--extra_params=DEPLOYMENT_SESSION_ID={deployment_id}") + cmd.append(f"--extra_params=DEPLOYMENT_SESSION_ID={deployment_id}") custom_params = getenv("CUSTOM_PARAMS") if custom_params: - logger.info(f"custom_params: {custom_params}") - cmdb_cli_cmd_call.append(f"--custom-params={custom_params}") - - if is_es_partial: - cleanup_mapping_path = effective_set_dir / ESGenerationContext.CLEANUP.value / ES_MAPPING_FILE - runtime_mapping_path = effective_set_dir / ESGenerationContext.RUNTIME.value / ES_MAPPING_FILE - deployment_mapping_path = effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ES_MAPPING_FILE - - cleanup_mapping = openYaml(cleanup_mapping_path) - runtime_mapping = openYaml(runtime_mapping_path) - deployment_mapping = openYaml(deployment_mapping_path) + cmd.append(f"--custom-params={custom_params}") - # run java effective set cli - subprocess.run(["sh", cmdb_cli_cmd_call], check=True) - - if is_es_partial: - new_cleanup_mapping = openYaml(cleanup_mapping_path) - new_runtime_mapping = openYaml(runtime_mapping_path) - new_deployment_mapping = openYaml(deployment_mapping_path) - - cleanup_mapping.update(new_cleanup_mapping) - runtime_mapping.update(new_runtime_mapping) - deployment_mapping.update(new_deployment_mapping) - - writeYamlToFile(cleanup_mapping_path, cleanup_mapping) - writeYamlToFile(runtime_mapping_path, runtime_mapping) - writeYamlToFile(deployment_mapping_path, deployment_mapping) - - encrypt_all_cred_files_for_env() + return cmd diff --git a/python/envgene/envgenehelper/sd_helper.py b/python/envgene/envgenehelper/sd_helper.py index 03bd2aab1..793218c5f 100644 --- a/python/envgene/envgenehelper/sd_helper.py +++ b/python/envgene/envgenehelper/sd_helper.py @@ -270,15 +270,12 @@ def get_sd_dir() -> Path: return Path(f'{get_current_env_dir_from_env_vars()}/{INVENTORY_DIR_NAME}/solution-descriptor/') -def resolve_sd_path() -> Path: - partial_gen = get_envgene_config_yaml().get("partial_effective_set_generation") +def resolve_delta_sd_path() -> Path: sd_dir = get_sd_dir() - if partial_gen: - logger.info("Partial effective set generation feature enabled") - merge_mode = calculate_merge_mode(getenv("SD_REPO_MERGE_MODE"), getenv('SD_DELTA')) - sd_version = getenv("SD_VERSION") - sd_data = getenv("SD_DATA") - sd_input = bool(sd_data) or bool(sd_version) - if sd_input and merge_mode in [MergeType.BASIC, MergeType.EXTENDED, MergeType.BASIC_EXCLUSION]: - return sd_dir.joinpath(DELTA_SD_FILE_NAME) - return sd_dir.joinpath(SD_FILE_NAME) + merge_mode = calculate_merge_mode(getenv("SD_REPO_MERGE_MODE"), getenv('SD_DELTA')) + sd_version = getenv("SD_VERSION") + sd_data = getenv("SD_DATA") + sd_input = bool(sd_data) or bool(sd_version) + if sd_input and merge_mode in [MergeType.BASIC, MergeType.EXTENDED, MergeType.BASIC_EXCLUSION]: + return sd_dir.joinpath(DELTA_SD_FILE_NAME) + From 4c7e242286c75ecd7248fcffee2f77a93475751c Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Wed, 6 May 2026 16:13:50 +0500 Subject: [PATCH 17/24] feat: refactor --- .../scripts/effective_set_entrypoint.py | 23 +++++++++++-------- build_pipegene/scripts/effective_set_job.py | 8 +++++-- python/envgene/envgenehelper/sd_helper.py | 12 +--------- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/build_effective_set_generator/scripts/effective_set_entrypoint.py b/build_effective_set_generator/scripts/effective_set_entrypoint.py index bf8a2a328..75c8dcb18 100644 --- a/build_effective_set_generator/scripts/effective_set_entrypoint.py +++ b/build_effective_set_generator/scripts/effective_set_entrypoint.py @@ -1,18 +1,18 @@ import os import subprocess from os import getenv -from pathlib import Path from envgenehelper import ( decrypt_all_cred_files_for_env, copy_path, validate_creds, openJson, encrypt_all_cred_files_for_env, resolve_sd_path, logger, get_current_env_dir_from_env_vars, cleanup_dir, get_envgene_config_yaml, openYaml, ESGenerationContext, - ES_MAPPING_FILE, writeYamlToFile, calculate_merge_mode, delete_dir, get_environment_name_from_full_name + ES_MAPPING_FILE, writeYamlToFile, calculate_merge_mode, + delete_dir, get_environment_name_from_full_name, + get_sd_dir, SD_FILE_NAME, MergeType, DELTA_SD_FILE_NAME ) from handle_effective_set_config import handle_effective_set_config -from python.envgene.envgenehelper import get_sd_dir, SD_FILE_NAME, MergeType from sboms_retention_policy import sboms_retention_policy @@ -33,8 +33,6 @@ def effective_set_entrypoint(): else: _run_full_generation(ctx) - encrypt_all_cred_files_for_env() - def _prepare_context(): full_env_name = getenv("FULL_ENV_NAME") @@ -42,8 +40,6 @@ def _prepare_context(): work_dir = os.path.join(ci_project_dir, "environments", full_env_name) merge_mode = calculate_merge_mode(getenv("SD_REPO_MERGE_MODE"), getenv("SD_DELTA")) - decrypt_all_cred_files_for_env() - artifact_app_defs_path = getenv("APP_DEFS_PATH") artifact_reg_defs_path = getenv("REG_DEFS_PATH") app_reg_defs_job = getenv("APP_REG_DEFS_JOB") @@ -52,11 +48,11 @@ def _prepare_context(): copy_path(artifact_app_defs_path, os.path.join(work_dir, "AppDefs")) copy_path(artifact_reg_defs_path, os.path.join(work_dir, "RegDefs")) - validate_creds() sboms_retention_policy() effective_set_dir = get_current_env_dir_from_env_vars() / "effective-set" - sd_path = resolve_sd_path() + sd_path = get_sd_dir().joinpath(SD_FILE_NAME) + delta_sd_path = get_sd_dir().joinpath(DELTA_SD_FILE_NAME) is_partial = get_envgene_config_yaml().get( "partial_effective_set_generation") and merge_mode.name != MergeType.REPLACE @@ -66,22 +62,27 @@ def _prepare_context(): "full_env_name": full_env_name, "effective_set_dir": effective_set_dir, "sd_path": sd_path, - "delta_sd_path": delta_sd_path + "delta_sd_path": delta_sd_path, "is_partial": is_partial, "is_reverse": is_reverse, } def _run_full_generation(ctx): + decrypt_all_cred_files_for_env() + validate_creds() effective_set_dir = ctx["effective_set_dir"] cmd = _build_cli_cmd(ctx) cleanup_dir(effective_set_dir) subprocess.run(["sh", cmd], check=True) + encrypt_all_cred_files_for_env() def _run_forward_merge(ctx): effective_set_dir = ctx["effective_set_dir"] delta_sd_path = ctx["delta_sd_path"] + decrypt_all_cred_files_for_env() + validate_creds() cmd = _build_cli_cmd(ctx) @@ -128,6 +129,8 @@ def _run_forward_merge(ctx): writeYamlToFile(runtime_mapping_path, runtime_mapping) writeYamlToFile(deployment_mapping_path, deployment_mapping) + encrypt_all_cred_files_for_env() + def _run_reverse_merge(ctx): effective_set_dir = ctx["effective_set_dir"] diff --git a/build_pipegene/scripts/effective_set_job.py b/build_pipegene/scripts/effective_set_job.py index 6693ce4c1..716dcb6e1 100644 --- a/build_pipegene/scripts/effective_set_job.py +++ b/build_pipegene/scripts/effective_set_job.py @@ -72,9 +72,13 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste def validate_topology_context_mode(effective_set_config_dict, full_env_name, params): effective_set_version = effective_set_config_dict.get("version") or "v2.0" - sd = bool(params["SD_DATA"]) or bool(params["SD_VERSION"]) + sd_input = bool(params["SD_DATA"]) or bool(params["SD_VERSION"]) full_sd_path = Path( f'{getenv('CI_PROJECT_DIR')}/environments/{full_env_name}/Inventory/solution-descriptor/sd.yaml') + any_sd = full_sd_path.exists() and sd_input # effective set generation in version 1.0 does not support no sd mode - if not (full_sd_path.exists() and sd) and effective_set_version.lower() == "v1.0": + if not any_sd and effective_set_version.lower() == "v1.0": raise ValueError("Feature generation effective set for pipeline and topology context is not supported for v1.0") + elif not any_sd: + logger.info("No-SD Mode: no SD present, only topology and pipeline contexts are generated; " + "deployment, runtime, and cleanup are skipped, SBOMs are not requested") diff --git a/python/envgene/envgenehelper/sd_helper.py b/python/envgene/envgenehelper/sd_helper.py index 793218c5f..999ea4595 100644 --- a/python/envgene/envgenehelper/sd_helper.py +++ b/python/envgene/envgenehelper/sd_helper.py @@ -6,6 +6,7 @@ SD_FILE_NAME = "sd.yaml" DELTA_SD_FILE_NAME = "delta_sd.yaml" + def get_app_name(name: str): return name[0:name.find(":")] @@ -268,14 +269,3 @@ def calculate_merge_mode(sd_merge_mode, sd_delta) -> MergeType: def get_sd_dir() -> Path: return Path(f'{get_current_env_dir_from_env_vars()}/{INVENTORY_DIR_NAME}/solution-descriptor/') - - -def resolve_delta_sd_path() -> Path: - sd_dir = get_sd_dir() - merge_mode = calculate_merge_mode(getenv("SD_REPO_MERGE_MODE"), getenv('SD_DELTA')) - sd_version = getenv("SD_VERSION") - sd_data = getenv("SD_DATA") - sd_input = bool(sd_data) or bool(sd_version) - if sd_input and merge_mode in [MergeType.BASIC, MergeType.EXTENDED, MergeType.BASIC_EXCLUSION]: - return sd_dir.joinpath(DELTA_SD_FILE_NAME) - From eaf5f0bd3ba172e81d03754f3a402f093790cfb6 Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Wed, 6 May 2026 18:01:34 +0500 Subject: [PATCH 18/24] feat: refactor --- .../scripts/crypt_manager.py | 26 ++++++++++++++++ .../scripts/effective_set_entrypoint.py | 12 -------- .../scripts/sboms_retention_policy.py | 4 +++ build_pipegene/scripts/effective_set_job.py | 30 ++++++++++++++----- 4 files changed, 53 insertions(+), 19 deletions(-) create mode 100644 build_effective_set_generator/scripts/crypt_manager.py diff --git a/build_effective_set_generator/scripts/crypt_manager.py b/build_effective_set_generator/scripts/crypt_manager.py new file mode 100644 index 000000000..ce2669fa6 --- /dev/null +++ b/build_effective_set_generator/scripts/crypt_manager.py @@ -0,0 +1,26 @@ +import click +from envgenehelper import encrypt_all_cred_files_for_env, decrypt_all_cred_files_for_env, validate_creds + + +@click.group(chain=True) +def crypt_manager(): + pass + + +@crypt_manager.command("decrypt_cred_files") +def decrypt_cred_files(): + decrypt_all_cred_files_for_env() + + +@crypt_manager.command("encrypt_cred_files") +def encrypt_cred_files(): + encrypt_all_cred_files_for_env() + + +@crypt_manager.command("validate_creds") +def validate_credentials(): + validate_creds() + + +if __name__ == "__main__": + crypt_manager() diff --git a/build_effective_set_generator/scripts/effective_set_entrypoint.py b/build_effective_set_generator/scripts/effective_set_entrypoint.py index 75c8dcb18..8ab69822a 100644 --- a/build_effective_set_generator/scripts/effective_set_entrypoint.py +++ b/build_effective_set_generator/scripts/effective_set_entrypoint.py @@ -36,20 +36,8 @@ def effective_set_entrypoint(): def _prepare_context(): full_env_name = getenv("FULL_ENV_NAME") - ci_project_dir = getenv("CI_PROJECT_DIR") - work_dir = os.path.join(ci_project_dir, "environments", full_env_name) merge_mode = calculate_merge_mode(getenv("SD_REPO_MERGE_MODE"), getenv("SD_DELTA")) - artifact_app_defs_path = getenv("APP_DEFS_PATH") - artifact_reg_defs_path = getenv("REG_DEFS_PATH") - app_reg_defs_job = getenv("APP_REG_DEFS_JOB") - - if artifact_app_defs_path and artifact_reg_defs_path and app_reg_defs_job: - copy_path(artifact_app_defs_path, os.path.join(work_dir, "AppDefs")) - copy_path(artifact_reg_defs_path, os.path.join(work_dir, "RegDefs")) - - sboms_retention_policy() - effective_set_dir = get_current_env_dir_from_env_vars() / "effective-set" sd_path = get_sd_dir().joinpath(SD_FILE_NAME) delta_sd_path = get_sd_dir().joinpath(DELTA_SD_FILE_NAME) diff --git a/build_effective_set_generator/scripts/sboms_retention_policy.py b/build_effective_set_generator/scripts/sboms_retention_policy.py index de19acbc9..2cb1a4447 100644 --- a/build_effective_set_generator/scripts/sboms_retention_policy.py +++ b/build_effective_set_generator/scripts/sboms_retention_policy.py @@ -28,3 +28,7 @@ def sboms_retention_policy(): cleanup_dir_by_age(app_sbom_dir, sbom_retention.keep_versions_per_app) cleanup_dir_by_size(sboms_dir, CI_JOB_ARTIFACT_MAX_SIZE_MB) + + +if __name__ == "__main__": + sboms_retention_policy() diff --git a/build_pipegene/scripts/effective_set_job.py b/build_pipegene/scripts/effective_set_job.py index 716dcb6e1..88142d15a 100644 --- a/build_pipegene/scripts/effective_set_job.py +++ b/build_pipegene/scripts/effective_set_job.py @@ -1,8 +1,9 @@ import json +import os from os import getenv, environ from pathlib import Path -from envgenehelper import logger +from envgenehelper import logger, copy_path from gcip import WhenStatement, Need from pipeline_helper import job_instance @@ -17,13 +18,20 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste effective_set_config_dict = json.loads(effective_set_config) validate_topology_context_mode(effective_set_config_dict, full_env_name, params) + app_reg_defs_job = params.get("APP_REG_DEFS_JOB") + is_local_app_def = init_local_app_defs_from_artifact(full_env_name, app_reg_defs_job, params) + script = [ # cert handling for java 'mkdir -p ${CI_PROJECT_DIR}/configuration/certs/', 'if [ -f /default_cert.pem ]; then cp /default_cert.pem "${CI_PROJECT_DIR}/configuration/certs/"; fi', 'for cert in "${CI_PROJECT_DIR}/configuration/certs/*" ; do [ -f "$cert" ] && keytool -import -trustcacerts -alias "$(basename "$cert")" -file "$cert" -keystore /etc/ssl/certs/keystore.jks -storepass changeit -noprompt; done', - 'python /scripts/effective_set_entrypoint.py' + 'python3 /module/scripts/crypt_manager.py decrypt_cred_files', + 'python3 /module/scripts/crypt_manager.py validate_creds', + 'python3 /module/scripts/sboms_retention_policy.py', + 'python3 /module/scripts/effective_set_entrypoint.py', + 'python3 /module/scripts/crypt_manager.py encrypt_cred_files', ] generate_effective_set_params = { @@ -43,10 +51,6 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste } needs = [] - app_reg_defs_job = params["APP_REG_DEFS_JOB"] - artifact_app_defs_path = params["APP_DEFS_PATH"] - artifact_reg_defs_path = params["REG_DEFS_PATH"] - is_local_app_def = artifact_app_defs_path and artifact_reg_defs_path and app_reg_defs_job if is_local_app_def: # gcip library doesn't allow to create a Need object that has the same pipeline as one it runs within. # We need to specify pipeline because generated job will be ran in child pipeline @@ -79,6 +83,18 @@ def validate_topology_context_mode(effective_set_config_dict, full_env_name, par # effective set generation in version 1.0 does not support no sd mode if not any_sd and effective_set_version.lower() == "v1.0": raise ValueError("Feature generation effective set for pipeline and topology context is not supported for v1.0") - elif not any_sd: + + if not any_sd: logger.info("No-SD Mode: no SD present, only topology and pipeline contexts are generated; " "deployment, runtime, and cleanup are skipped, SBOMs are not requested") + + +def init_local_app_defs_from_artifact(full_env_name, app_reg_defs_job, params): + work_dir = os.path.join(getenv('CI_PROJECT_DIR'), "environments", full_env_name) + artifact_app_defs_path = params.get("APP_DEFS_PATH") + artifact_reg_defs_path = params.get("REG_DEFS_PATH") + is_local_app_def = artifact_app_defs_path and artifact_reg_defs_path and app_reg_defs_job + if is_local_app_def: + copy_path(artifact_app_defs_path, os.path.join(work_dir, "AppDefs")) + copy_path(artifact_reg_defs_path, os.path.join(work_dir, "RegDefs")) + return is_local_app_def From 57437d147e63f72b5b4e7d53c77641c67d86ad21 Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Wed, 6 May 2026 18:39:40 +0500 Subject: [PATCH 19/24] feat: refactor --- .../scripts/effective_set_entrypoint.py | 80 ++++++------------- python/envgene/envgenehelper/constants.py | 11 --- .../envgenehelper/effective_set_helper.py | 47 +++++++++++ 3 files changed, 70 insertions(+), 68 deletions(-) create mode 100644 python/envgene/envgenehelper/effective_set_helper.py diff --git a/build_effective_set_generator/scripts/effective_set_entrypoint.py b/build_effective_set_generator/scripts/effective_set_entrypoint.py index 8ab69822a..0a269e41e 100644 --- a/build_effective_set_generator/scripts/effective_set_entrypoint.py +++ b/build_effective_set_generator/scripts/effective_set_entrypoint.py @@ -1,78 +1,50 @@ -import os import subprocess from os import getenv from envgenehelper import ( - decrypt_all_cred_files_for_env, copy_path, validate_creds, - openJson, encrypt_all_cred_files_for_env, resolve_sd_path, - logger, get_current_env_dir_from_env_vars, cleanup_dir, - get_envgene_config_yaml, openYaml, ESGenerationContext, - ES_MAPPING_FILE, writeYamlToFile, calculate_merge_mode, - delete_dir, get_environment_name_from_full_name, - get_sd_dir, SD_FILE_NAME, MergeType, DELTA_SD_FILE_NAME + decrypt_all_cred_files_for_env, validate_creds, + openJson, encrypt_all_cred_files_for_env, get_current_env_dir_from_env_vars, cleanup_dir, + openYaml, ESGenerationContext, resolve_partial_merge_mode, PartialMergeMode, + ES_MAPPING_FILE, writeYamlToFile, delete_dir, get_environment_name_from_full_name, + get_sd_dir, SD_FILE_NAME, DELTA_SD_FILE_NAME, GenerationMode, resolve_es_generation_mode ) from handle_effective_set_config import handle_effective_set_config -from sboms_retention_policy import sboms_retention_policy def effective_set_entrypoint(): - ctx = _prepare_context() + full_env_name = getenv("FULL_ENV_NAME") + effective_set_dir = get_current_env_dir_from_env_vars() / "effective-set" + sd_path = get_sd_dir().joinpath(SD_FILE_NAME) + delta_sd_path = get_sd_dir().joinpath(DELTA_SD_FILE_NAME) - if ctx["is_partial"]: - delta_sd_path = ctx["delta_sd_path"] - sd_path = ctx["sd_path"].exists() - if not ctx["delta_sd_path"].exists() and ctx["sd_path"].exists(): + if resolve_es_generation_mode() == GenerationMode.PARTIAL: + if not delta_sd_path.exists() and sd_path.exists(): raise ValueError( f"[Partial effective set generation] impossible without delta sd {delta_sd_path} and full sd {sd_path}") - elif ctx["is_reverse"]: - _run_reverse_merge(ctx) + elif resolve_partial_merge_mode() == PartialMergeMode.REVERSE: + _run_reverse_merge(effective_set_dir, full_env_name, delta_sd_path) else: - _run_forward_merge(ctx) + _run_forward_merge(effective_set_dir, full_env_name, delta_sd_path) else: - _run_full_generation(ctx) + _run_full_generation(effective_set_dir, full_env_name, sd_path) -def _prepare_context(): - full_env_name = getenv("FULL_ENV_NAME") - merge_mode = calculate_merge_mode(getenv("SD_REPO_MERGE_MODE"), getenv("SD_DELTA")) - - effective_set_dir = get_current_env_dir_from_env_vars() / "effective-set" - sd_path = get_sd_dir().joinpath(SD_FILE_NAME) - delta_sd_path = get_sd_dir().joinpath(DELTA_SD_FILE_NAME) - - is_partial = get_envgene_config_yaml().get( - "partial_effective_set_generation") and merge_mode.name != MergeType.REPLACE - is_reverse = is_partial and merge_mode.name == MergeType.BASIC_EXCLUSION - - return { - "full_env_name": full_env_name, - "effective_set_dir": effective_set_dir, - "sd_path": sd_path, - "delta_sd_path": delta_sd_path, - "is_partial": is_partial, - "is_reverse": is_reverse, - } - - -def _run_full_generation(ctx): +def _run_full_generation(effective_set_dir, full_env_name, sd_path): decrypt_all_cred_files_for_env() validate_creds() - effective_set_dir = ctx["effective_set_dir"] - cmd = _build_cli_cmd(ctx) + cmd = _build_cli_cmd(effective_set_dir, full_env_name, sd_path) cleanup_dir(effective_set_dir) subprocess.run(["sh", cmd], check=True) encrypt_all_cred_files_for_env() -def _run_forward_merge(ctx): - effective_set_dir = ctx["effective_set_dir"] - delta_sd_path = ctx["delta_sd_path"] +def _run_forward_merge(effective_set_dir, full_env_name, delta_sd_path): decrypt_all_cred_files_for_env() validate_creds() - cmd = _build_cli_cmd(ctx) + cmd = _build_cli_cmd(effective_set_dir, full_env_name, delta_sd_path) delta_sd = openYaml(delta_sd_path) apps = delta_sd.get("applications", []) @@ -120,10 +92,7 @@ def _run_forward_merge(ctx): encrypt_all_cred_files_for_env() -def _run_reverse_merge(ctx): - effective_set_dir = ctx["effective_set_dir"] - delta_sd_path = ctx["delta_sd_path"] - +def _run_reverse_merge(effective_set_dir, full_env_name, delta_sd_path): apps = openYaml(delta_sd_path).get("applications", []) for app in apps: @@ -142,7 +111,7 @@ def _run_reverse_merge(ctx): delete_dir(cleanup_dp) delete_dir(deployment_dp) - ns = get_environment_name_from_full_name(ctx["full_env_name"]) + ns = get_environment_name_from_full_name(full_env_name) mapping_paths = [ effective_set_dir / ESGenerationContext.CLEANUP.value / ES_MAPPING_FILE, effective_set_dir / ESGenerationContext.RUNTIME.value / ES_MAPPING_FILE, @@ -155,13 +124,10 @@ def _run_reverse_merge(ctx): writeYamlToFile(path, mapping) -def _build_cli_cmd(ctx): - effective_set_dir = ctx["effective_set_dir"] - sd_path = ctx["sd_path"] - +def _build_cli_cmd(effective_set_dir, full_env_name, sd_path): cmd = [ "/module/scripts/utils/run_effective_set_cli.sh", - f"--env-id={ctx['full_env_name']}", + f"--env-id={full_env_name}", "--envs-path=$CI_PROJECT_DIR/environments", f"--output={effective_set_dir}", ] diff --git a/python/envgene/envgenehelper/constants.py b/python/envgene/envgenehelper/constants.py index 5699b8c3b..a68a33b8d 100644 --- a/python/envgene/envgenehelper/constants.py +++ b/python/envgene/envgenehelper/constants.py @@ -12,14 +12,3 @@ ] CI_JOB_ARTIFACT_MAX_SIZE_MB = 1200 # 80% from limit 1.5 - - -class ESGenerationContext(Enum): - TOPOLOGY = "topology" - PIPELINE = "pipeline" - DEPLOYMENT = "deployment" - RUNTIME = "runtime" - CLEANUP = "cleanup" - - -ES_MAPPING_FILE = "mapping.yaml" diff --git a/python/envgene/envgenehelper/effective_set_helper.py b/python/envgene/envgenehelper/effective_set_helper.py new file mode 100644 index 000000000..63132f2a1 --- /dev/null +++ b/python/envgene/envgenehelper/effective_set_helper.py @@ -0,0 +1,47 @@ +from enum import Enum +from os import getenv + +from envgenehelper import get_envgene_config_yaml, calculate_merge_mode, MergeType + + +class ESGenerationContext(Enum): + TOPOLOGY = "topology" + PIPELINE = "pipeline" + DEPLOYMENT = "deployment" + RUNTIME = "runtime" + CLEANUP = "cleanup" + + +ES_MAPPING_FILE = "mapping.yaml" + + +class GenerationMode(Enum): + FULL = "full" + PARTIAL = "partial" + NO_SD = "no_sd" + + +class PartialMergeMode(Enum): + FORWARD = "forward" + REVERSE = "reverse" + + +def resolve_partial_merge_mode(): + merge_mode = calculate_merge_mode(getenv("SD_REPO_MERGE_MODE"), getenv("SD_DELTA")) + + if merge_mode == MergeType.BASIC_EXCLUSION: + return PartialMergeMode.REVERSE + + if merge_mode in {MergeType.BASIC, MergeType.EXTENDED}: + return PartialMergeMode.FORWARD + + raise ValueError(f"Unsupported merge mode for partial: {merge_mode}") + + +def resolve_es_generation_mode(): + merge_mode = calculate_merge_mode(getenv("SD_REPO_MERGE_MODE"), getenv("SD_DELTA")) + + if get_envgene_config_yaml().get("partial_effective_set_generation") and merge_mode.name != MergeType.REPLACE: + return GenerationMode.PARTIAL + + return GenerationMode.FULL From 845defd08b92f18963cae86f6114d310316f076f Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Wed, 6 May 2026 19:14:59 +0500 Subject: [PATCH 20/24] feat: delete delta sd --- .../scripts/effective_set_entrypoint.py | 26 ++++++++----------- python/envgene/envgenehelper/file_helper.py | 5 ++-- scripts/build_env/process_sd.py | 4 ++- 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/build_effective_set_generator/scripts/effective_set_entrypoint.py b/build_effective_set_generator/scripts/effective_set_entrypoint.py index 0a269e41e..02bff115d 100644 --- a/build_effective_set_generator/scripts/effective_set_entrypoint.py +++ b/build_effective_set_generator/scripts/effective_set_entrypoint.py @@ -1,13 +1,14 @@ import subprocess from os import getenv -from envgenehelper import ( - decrypt_all_cred_files_for_env, validate_creds, - openJson, encrypt_all_cred_files_for_env, get_current_env_dir_from_env_vars, cleanup_dir, - openYaml, ESGenerationContext, resolve_partial_merge_mode, PartialMergeMode, - ES_MAPPING_FILE, writeYamlToFile, delete_dir, get_environment_name_from_full_name, - get_sd_dir, SD_FILE_NAME, DELTA_SD_FILE_NAME, GenerationMode, resolve_es_generation_mode -) +from envgenehelper.business_helper import get_environment_name_from_full_name, get_current_env_dir_from_env_vars +from envgenehelper.file_helper import writeYamlToFile, delete_dir, deleteFileIfExists, openYaml, cleanup_dir +from envgenehelper.json_helper import openJson +from envgenehelper.logger import logger +from envgenehelper.yaml_helper import writeYamlToFile +from envgenehelper.effective_set_helper import resolve_es_generation_mode, GenerationMode, PartialMergeMode, \ + resolve_partial_merge_mode, ES_MAPPING_FILE +from envgenehelper.sd_helper import get_sd_dir, DELTA_SD_FILE_NAME, SD_FILE_NAME from handle_effective_set_config import handle_effective_set_config @@ -30,20 +31,17 @@ def effective_set_entrypoint(): else: _run_full_generation(effective_set_dir, full_env_name, sd_path) + # do not commit delta sd to repo + deleteFileIfExists(delta_sd_path) + def _run_full_generation(effective_set_dir, full_env_name, sd_path): - decrypt_all_cred_files_for_env() - validate_creds() cmd = _build_cli_cmd(effective_set_dir, full_env_name, sd_path) cleanup_dir(effective_set_dir) subprocess.run(["sh", cmd], check=True) - encrypt_all_cred_files_for_env() def _run_forward_merge(effective_set_dir, full_env_name, delta_sd_path): - decrypt_all_cred_files_for_env() - validate_creds() - cmd = _build_cli_cmd(effective_set_dir, full_env_name, delta_sd_path) delta_sd = openYaml(delta_sd_path) @@ -89,8 +87,6 @@ def _run_forward_merge(effective_set_dir, full_env_name, delta_sd_path): writeYamlToFile(runtime_mapping_path, runtime_mapping) writeYamlToFile(deployment_mapping_path, deployment_mapping) - encrypt_all_cred_files_for_env() - def _run_reverse_merge(effective_set_dir, full_env_name, delta_sd_path): apps = openYaml(delta_sd_path).get("applications", []) diff --git a/python/envgene/envgenehelper/file_helper.py b/python/envgene/envgenehelper/file_helper.py index 6b547612c..be006f96e 100644 --- a/python/envgene/envgenehelper/file_helper.py +++ b/python/envgene/envgenehelper/file_helper.py @@ -1,12 +1,11 @@ -import os import glob +import os import re import shutil import tarfile -import time import zipfile -from typing import Callable from pathlib import Path +from typing import Callable from .logger import logger diff --git a/scripts/build_env/process_sd.py b/scripts/build_env/process_sd.py index d756d643a..7dc6c51d2 100644 --- a/scripts/build_env/process_sd.py +++ b/scripts/build_env/process_sd.py @@ -12,7 +12,7 @@ from envgenehelper.business_helper import getenv_and_log, getenv_with_error from envgenehelper.collections_helper import split_multi_value_param from envgenehelper.env_helper import Environment -from envgenehelper.file_helper import identify_yaml_extension +from envgenehelper.file_helper import identify_yaml_extension, deleteFileIfExists from envgenehelper.logger import logger from envgenehelper.plugin_engine import PluginEngine from envgenehelper.sd_helper import basic_merge_multiple, MergeType, calculate_merge_mode @@ -157,6 +157,8 @@ def handle_sd(env, sd_source_type, sd_version, sd_data, sd_delta, sd_merge_mode) effective_merge_mode = calculate_merge_mode(sd_merge_mode, sd_delta) helper.check_dir_exist_and_create(base_sd_path) + # do not commit delta sd to repo, delete old ones + deleteFileIfExists(base_sd_path.joinpath(DELTA_SD_FILE_NAME)) if sd_source_type == "artifact": download_sds_with_version(env, base_sd_path, sd_version, effective_merge_mode) elif sd_source_type == "json": From 5d3e7c22dbcc37db8601bf5769be97609d9386a7 Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Wed, 6 May 2026 20:55:53 +0500 Subject: [PATCH 21/24] feat: fixes --- .../scripts/effective_set_entrypoint.py | 79 +++++++++++-------- python/envgene/envgenehelper/file_helper.py | 5 ++ 2 files changed, 50 insertions(+), 34 deletions(-) diff --git a/build_effective_set_generator/scripts/effective_set_entrypoint.py b/build_effective_set_generator/scripts/effective_set_entrypoint.py index 02bff115d..3279677cf 100644 --- a/build_effective_set_generator/scripts/effective_set_entrypoint.py +++ b/build_effective_set_generator/scripts/effective_set_entrypoint.py @@ -2,12 +2,12 @@ from os import getenv from envgenehelper.business_helper import get_environment_name_from_full_name, get_current_env_dir_from_env_vars -from envgenehelper.file_helper import writeYamlToFile, delete_dir, deleteFileIfExists, openYaml, cleanup_dir +from envgenehelper.file_helper import delete_dir, deleteFileIfExists, delete_dir_if_exists from envgenehelper.json_helper import openJson from envgenehelper.logger import logger -from envgenehelper.yaml_helper import writeYamlToFile +from envgenehelper.yaml_helper import writeYamlToFile, openYaml from envgenehelper.effective_set_helper import resolve_es_generation_mode, GenerationMode, PartialMergeMode, \ - resolve_partial_merge_mode, ES_MAPPING_FILE + resolve_partial_merge_mode, ES_MAPPING_FILE, ESGenerationContext from envgenehelper.sd_helper import get_sd_dir, DELTA_SD_FILE_NAME, SD_FILE_NAME from handle_effective_set_config import handle_effective_set_config @@ -23,6 +23,9 @@ def effective_set_entrypoint(): if not delta_sd_path.exists() and sd_path.exists(): raise ValueError( f"[Partial effective set generation] impossible without delta sd {delta_sd_path} and full sd {sd_path}") + # first run + if not effective_set_dir.exists(): + _run_full_generation(effective_set_dir, full_env_name, sd_path) elif resolve_partial_merge_mode() == PartialMergeMode.REVERSE: _run_reverse_merge(effective_set_dir, full_env_name, delta_sd_path) @@ -37,7 +40,7 @@ def effective_set_entrypoint(): def _run_full_generation(effective_set_dir, full_env_name, sd_path): cmd = _build_cli_cmd(effective_set_dir, full_env_name, sd_path) - cleanup_dir(effective_set_dir) + delete_dir(effective_set_dir) subprocess.run(["sh", cmd], check=True) @@ -54,30 +57,30 @@ def _run_forward_merge(effective_set_dir, full_env_name, delta_sd_path): } for ns in deploy_postfixes: - cleanup_dir(effective_set_dir / ESGenerationContext.CLEANUP.value / ns) + delete_dir_if_exists(effective_set_dir / ESGenerationContext.CLEANUP.value / ns) for app in apps: app_name = app.get("version", "").split(":")[0] ns = app.get("deployPostfix") - cleanup_dir(effective_set_dir / ESGenerationContext.RUNTIME.value / ns / app_name) - cleanup_dir(effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ns / app_name) + delete_dir_if_exists(effective_set_dir / ESGenerationContext.RUNTIME.value / ns / app_name) + delete_dir_if_exists(effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ns / app_name) - cleanup_dir(effective_set_dir / ESGenerationContext.TOPOLOGY.value) - cleanup_dir(effective_set_dir / ESGenerationContext.PIPELINE.value) + delete_dir_if_exists(effective_set_dir / ESGenerationContext.TOPOLOGY.value) + delete_dir_if_exists(effective_set_dir / ESGenerationContext.PIPELINE.value) cleanup_mapping_path = effective_set_dir / ESGenerationContext.CLEANUP.value / ES_MAPPING_FILE runtime_mapping_path = effective_set_dir / ESGenerationContext.RUNTIME.value / ES_MAPPING_FILE deployment_mapping_path = effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ES_MAPPING_FILE - cleanup_mapping = openYaml(cleanup_mapping_path) - runtime_mapping = openYaml(runtime_mapping_path) - deployment_mapping = openYaml(deployment_mapping_path) + cleanup_mapping = openYaml(cleanup_mapping_path, allow_default=True) + runtime_mapping = openYaml(runtime_mapping_path, allow_default=True) + deployment_mapping = openYaml(deployment_mapping_path, allow_default=True) subprocess.run(["sh", cmd], check=True) - new_cleanup_mapping = openYaml(cleanup_mapping_path) - new_runtime_mapping = openYaml(runtime_mapping_path) - new_deployment_mapping = openYaml(deployment_mapping_path) + new_cleanup_mapping = openYaml(cleanup_mapping_path, allow_default=True) + new_runtime_mapping = openYaml(runtime_mapping_path, allow_default=True) + new_deployment_mapping = openYaml(deployment_mapping_path, allow_default=True) cleanup_mapping.update(new_cleanup_mapping) runtime_mapping.update(new_runtime_mapping) @@ -91,6 +94,13 @@ def _run_forward_merge(effective_set_dir, full_env_name, delta_sd_path): def _run_reverse_merge(effective_set_dir, full_env_name, delta_sd_path): apps = openYaml(delta_sd_path).get("applications", []) + mapping_paths = [ + effective_set_dir / ESGenerationContext.CLEANUP.value / ES_MAPPING_FILE, + effective_set_dir / ESGenerationContext.RUNTIME.value / ES_MAPPING_FILE, + effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ES_MAPPING_FILE, + ] + ns = get_environment_name_from_full_name(full_env_name) + for app in apps: app_name = app.get("version", "").split(":")[0] dp = app.get("deployPostfix") @@ -99,25 +109,26 @@ def _run_reverse_merge(effective_set_dir, full_env_name, delta_sd_path): deployment_dp = effective_set_dir / ESGenerationContext.DEPLOYMENT.value / dp cleanup_dp = effective_set_dir / ESGenerationContext.CLEANUP.value / dp - cleanup_dir(runtime_dp / app_name) - cleanup_dir(deployment_dp / app_name) - - if not any(runtime_dp.iterdir()): - delete_dir(runtime_dp) - delete_dir(cleanup_dp) - delete_dir(deployment_dp) - - ns = get_environment_name_from_full_name(full_env_name) - mapping_paths = [ - effective_set_dir / ESGenerationContext.CLEANUP.value / ES_MAPPING_FILE, - effective_set_dir / ESGenerationContext.RUNTIME.value / ES_MAPPING_FILE, - effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ES_MAPPING_FILE, - ] - for path in mapping_paths: - mapping = openYaml(path, allow_default=True) - if ns in mapping: - mapping.pop(ns) - writeYamlToFile(path, mapping) + delete_dir(runtime_dp / app_name) + delete_dir(deployment_dp / app_name) + + if any(runtime_dp.iterdir()): + continue + + delete_dir(runtime_dp) + delete_dir(deployment_dp) + delete_dir(cleanup_dp) + + for path in mapping_paths: + if not path.exists(): + logger.warning(f"Mapping file not found, skipping: {path}") + continue + mapping = openYaml(path, allow_default=True) or {} + if ns not in mapping: + logger.warning(f"Namespace '{ns}' not found in mapping file {path}, skipping removing by key: {ns}") + continue + mapping.pop(ns) + writeYamlToFile(path, mapping) def _build_cli_cmd(effective_set_dir, full_env_name, sd_path): diff --git a/python/envgene/envgenehelper/file_helper.py b/python/envgene/envgenehelper/file_helper.py index be006f96e..56ca7e27b 100644 --- a/python/envgene/envgenehelper/file_helper.py +++ b/python/envgene/envgenehelper/file_helper.py @@ -302,3 +302,8 @@ def cleanup_dir_by_age(dir_path, keep_last: int): for old_file in files[keep_last:]: logger.info(f"Removing file: {old_file}") deleteFileIfExists(old_file) + + +def delete_dir_if_exists(dir_path): + if os.path.isdir(dir_path): + shutil.rmtree(dir_path) From e148a69aa8836361bcf24a50285efa8a7b5862f2 Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Tue, 12 May 2026 17:32:01 +0500 Subject: [PATCH 22/24] feat: fixes --- scripts/build_env/process_sd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_env/process_sd.py b/scripts/build_env/process_sd.py index 7dc6c51d2..54713ff01 100644 --- a/scripts/build_env/process_sd.py +++ b/scripts/build_env/process_sd.py @@ -8,14 +8,14 @@ import yaml from artifact_searcher import artifact from artifact_searcher.utils import models as artifact_models -from envgenehelper import get_sd_dir, SD_FILE_NAME, DELTA_SD_FILE_NAME from envgenehelper.business_helper import getenv_and_log, getenv_with_error from envgenehelper.collections_helper import split_multi_value_param from envgenehelper.env_helper import Environment from envgenehelper.file_helper import identify_yaml_extension, deleteFileIfExists from envgenehelper.logger import logger from envgenehelper.plugin_engine import PluginEngine -from envgenehelper.sd_helper import basic_merge_multiple, MergeType, calculate_merge_mode +from envgenehelper.sd_helper import (basic_merge_multiple, MergeType, calculate_merge_mode, + get_sd_dir, SD_FILE_NAME, DELTA_SD_FILE_NAME) from typing_extensions import deprecated MERGE_METHODS = { @@ -120,7 +120,7 @@ def merge_sd(sd_path: Path, sd_data, merge_func): logger.info(f"Merged data into Target Path! - {result}") -@deprecated +@deprecated("SD_DELTA is deprecated") def calculate_sd_delta(sd_delta): logger.info(f"printing sd_delta before {sd_delta}") if sd_delta is not None and str(sd_delta).strip() != "": From f73165f8eef56b490ab07e47545507b89b5fb562 Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Tue, 12 May 2026 19:20:45 +0500 Subject: [PATCH 23/24] feat: tests --- scripts/build_env/tests/base_test.py | 1 + .../tests/env-build/test_paramset_sorting.py | 1 - .../tests/sd/test_process_sd_artifact.py | 100 +++++++++++------- 3 files changed, 60 insertions(+), 42 deletions(-) diff --git a/scripts/build_env/tests/base_test.py b/scripts/build_env/tests/base_test.py index e170370a5..611dbdf86 100644 --- a/scripts/build_env/tests/base_test.py +++ b/scripts/build_env/tests/base_test.py @@ -9,6 +9,7 @@ class BaseTest: expected_dir = test_data_dir os.environ['CI_PROJECT_DIR'] = str(test_data_dir) os.environ['JSON_SCHEMAS_DIR'] = str(base_dir / "schemas") + output_dir = base_dir / "tmp" def set_ci_project_dir(self, *subdirs) -> Path: self.ci_project_dir = self.ci_project_dir.joinpath(*subdirs) diff --git a/scripts/build_env/tests/env-build/test_paramset_sorting.py b/scripts/build_env/tests/env-build/test_paramset_sorting.py index e88b7d102..cc286d0a2 100644 --- a/scripts/build_env/tests/env-build/test_paramset_sorting.py +++ b/scripts/build_env/tests/env-build/test_paramset_sorting.py @@ -1,4 +1,3 @@ -from envgenehelper.business_helper import is_from_template_dir from build_env import sort_paramsets_with_same_name class TestSortParamsetsWithSameName: diff --git a/scripts/build_env/tests/sd/test_process_sd_artifact.py b/scripts/build_env/tests/sd/test_process_sd_artifact.py index 071308c21..9023c9697 100644 --- a/scripts/build_env/tests/sd/test_process_sd_artifact.py +++ b/scripts/build_env/tests/sd/test_process_sd_artifact.py @@ -4,6 +4,7 @@ from unittest.mock import patch from ruamel.yaml import YAML +from scripts.build_env.tests.base_test import BaseTest from test_sd_helpers import do_prerequisites, assert_sd_contents, load_test_pipeline_sd_data os.environ['ENVIRONMENT_NAME'] = "temporary" @@ -32,49 +33,66 @@ "replace": [] } -TEST_SD_DIR = Path(getAbsPath("../../test_data/test_handle_sd")) -OUTPUT_DIR = getAbsPath("../../tmp/test_handle_sd") SD = "sd.yaml" +FEATURE_TEST_DIR = "test_handle_sd" + + +class TestSdProcessArtifact(BaseTest): + env_dir = "" + cluster_dir = "" + full_env_name = "cluster-01/env-01" + + def setup_method(self): + self.set_ci_project_dir(FEATURE_TEST_DIR) + self.test_data_dir = self.test_data_dir / FEATURE_TEST_DIR + self.output_dir = self.output_dir / FEATURE_TEST_DIR + self.env_name = get_environment_name_from_full_name(self.full_env_name) + self.cluster = get_cluster_name_from_full_name(self.full_env_name) + os.environ["FULL_ENV_NAME"] = self.full_env_name + os.environ["ENV_NAME"] = self.env_name + os.environ["CLUSTER_NAME"] = self.cluster + + @pytest.mark.parametrize("test_case_name", TEST_CASES_POSITIVE) + @patch("process_sd.download_sd_by_appver") + def test_sd_positive(self, mock_download_sd, test_case_name): + env = Environment(str(Path(self.output_dir, test_case_name)), self.cluster, self.env_name) + do_prerequisites(SD, self.test_data_dir, self.output_dir, test_case_name, env, test_suits_map) + logger.info(f"======TEST HANDLE_SD_ARTIFACT_POSITIVE: {test_case_name}======") + logger.info(f"Starting SD test:" + f"\n\tTest case: {test_case_name}") + + sd_data, sd_source_type, sd_version, sd_delta, sd_merge_mode = load_test_pipeline_sd_data(self.test_data_dir, + test_case_name) + + file_path = Path(self.test_data_dir, test_case_name, "mock_sd.json") + sd_data = openJson(file_path) + mock_download_sd.return_value = sd_data -@pytest.mark.parametrize("test_case_name", TEST_CASES_POSITIVE) -@patch("process_sd.download_sd_by_appver") -def test_sd_positive(mock_download_sd, test_case_name): - env = Environment(str(Path(OUTPUT_DIR, test_case_name)), "cluster-01", "env-01") - do_prerequisites(SD, TEST_SD_DIR, OUTPUT_DIR, test_case_name, env, test_suits_map) - logger.info(f"======TEST HANDLE_SD_ARTIFACT_POSITIVE: {test_case_name}======") - logger.info(f"Starting SD test:" - f"\n\tTest case: {test_case_name}") - - sd_data, sd_source_type, sd_version, sd_delta, sd_merge_mode = load_test_pipeline_sd_data(TEST_SD_DIR, test_case_name) - - file_path = Path(TEST_SD_DIR, test_case_name, f"mock_sd.json") - sd_data = openJson(file_path) - mock_download_sd.return_value = sd_data - - handle_sd(env, sd_source_type, sd_version, sd_data, sd_delta, sd_merge_mode) - actual_dir = os.path.join(env.env_path, "Inventory", "solution-descriptor") - - assert_sd_contents(TEST_SD_DIR, OUTPUT_DIR, test_case_name, actual_dir, test_suits_map) - logger.info(f"=====SUCCESS - {test_case_name}======") - - -@pytest.mark.parametrize("test_case_name,expected_exception", [(k, v) for k, v in TEST_CASES_NEGATIVE.items()]) -@patch("process_sd.download_sd_by_appver") -def test_sd_negative(mock_download_sd, test_case_name, expected_exception): - env = Environment(str(Path(OUTPUT_DIR, test_case_name)), "cluster-01", "env-01") - do_prerequisites(SD, TEST_SD_DIR, OUTPUT_DIR, test_case_name, env, test_suits_map) - logger.info(f"======TEST HANDLE_SD_ARTIFACT_NEGATIVE: {test_case_name}======") - logger.info(f"Starting SD test:" - f"\n\tTest case: {test_case_name}") - - sd_data, sd_source_type, sd_version, sd_delta, sd_merge_mode = load_test_pipeline_sd_data(TEST_SD_DIR, test_case_name) - - file_path = Path(TEST_SD_DIR, test_case_name, f"mock_sd.json") - sd_data = openJson(file_path) - mock_download_sd.return_value = sd_data - - with pytest.raises(expected_exception): handle_sd(env, sd_source_type, sd_version, sd_data, sd_delta, sd_merge_mode) + actual_dir = os.path.join(env.env_path, "Inventory", "solution-descriptor") + + assert_sd_contents(self.test_data_dir, self.output_dir, test_case_name, actual_dir, test_suits_map) + logger.info(f"=====SUCCESS - {test_case_name}======") + + @pytest.mark.parametrize("test_case_name,expected_exception", [(k, v) for k, v in TEST_CASES_NEGATIVE.items()]) + @patch("process_sd.download_sd_by_appver") + def test_sd_negative(self, mock_download_sd, test_case_name, expected_exception): + + env = Environment(str(Path(self.output_dir, test_case_name)), self.cluster, self.env_name) + do_prerequisites(SD, self.test_data_dir, self.output_dir, test_case_name, env, test_suits_map) + logger.info(f"======TEST HANDLE_SD_ARTIFACT_NEGATIVE: {test_case_name}======") + logger.info(f"Starting SD test:" + f"\n\tTest case: {test_case_name}") + + sd_data, sd_source_type, sd_version, sd_delta, sd_merge_mode = load_test_pipeline_sd_data(self.test_data_dir, + test_case_name) + + file_path = Path(self.test_data_dir, test_case_name, "mock_sd.json") + sd_data = openJson(file_path) + mock_download_sd.return_value = sd_data + + with pytest.raises(expected_exception): + handle_sd(env, sd_source_type, sd_version, sd_data, sd_delta, sd_merge_mode) - logger.info(f"=====SUCCESS - {test_case_name}======") + logger.info(f"=====SUCCESS - {test_case_name}======") From 5ca0415aa6585870efac0e1b68044e2c79573a5b Mon Sep 17 00:00:00 2001 From: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Date: Tue, 12 May 2026 20:50:36 +0500 Subject: [PATCH 24/24] feat: tests --- scripts/build_env/tests/base_test.py | 4 +- .../test_env_inv_generation.py | 4 +- .../tests/sd/test_process_sd_artifact.py | 25 ++++----- .../tests/sd/test_process_sd_local.py | 53 ++++++++++++------- scripts/build_env/tests/sd/test_sd_helpers.py | 2 + .../ER/basic_not_first/delta_sd.yaml | 5 -- .../test_handle_sd/ER/exclude/delta_sd.yaml | 5 -- .../test_handle_sd/ER/extended/delta_sd.yaml | 19 ------- 8 files changed, 50 insertions(+), 67 deletions(-) delete mode 100644 test_data/test_handle_sd/ER/basic_not_first/delta_sd.yaml delete mode 100644 test_data/test_handle_sd/ER/exclude/delta_sd.yaml delete mode 100644 test_data/test_handle_sd/ER/extended/delta_sd.yaml diff --git a/scripts/build_env/tests/base_test.py b/scripts/build_env/tests/base_test.py index 611dbdf86..36200e358 100644 --- a/scripts/build_env/tests/base_test.py +++ b/scripts/build_env/tests/base_test.py @@ -5,11 +5,11 @@ class BaseTest: base_dir = Path(__file__).resolve().parents[3] test_data_dir = base_dir / "test_data" + output_dir = base_dir / "tmp" + ci_project_dir = test_data_dir - expected_dir = test_data_dir os.environ['CI_PROJECT_DIR'] = str(test_data_dir) os.environ['JSON_SCHEMAS_DIR'] = str(base_dir / "schemas") - output_dir = base_dir / "tmp" def set_ci_project_dir(self, *subdirs) -> Path: self.ci_project_dir = self.ci_project_dir.joinpath(*subdirs) diff --git a/scripts/build_env/tests/env_inventory_generation/test_env_inv_generation.py b/scripts/build_env/tests/env_inventory_generation/test_env_inv_generation.py index 871fe92ed..89c5f3b01 100644 --- a/scripts/build_env/tests/env_inventory_generation/test_env_inv_generation.py +++ b/scripts/build_env/tests/env_inventory_generation/test_env_inv_generation.py @@ -3,7 +3,7 @@ import yaml from env_inventory_generation import generate_env_new_approach, Place, resolve_path, INVENTORY, Action -from envgenehelper import get_cluster_name_from_full_name, dumpYamlToStr, get_environment_name_from_full_name, readYaml, \ +from envgenehelper import get_cluster_name_from_full_name, get_environment_name_from_full_name, readYaml, \ is_dir_empty, writeYamlToFile from envgenehelper.test_helpers import TestHelpers from jinja.jinja import create_jinja_env @@ -22,7 +22,7 @@ class TestEnvInvGen(BaseTest): def setup_method(self): self.set_ci_project_dir(FEATURE_TEST_DIR, "output") - self.expected_dir = self.expected_dir.joinpath(FEATURE_TEST_DIR, "expected") + self.expected_dir = self.test_data_dir / FEATURE_TEST_DIR / "expected" TestHelpers.clean_test_dir(self.ci_project_dir) self.env_name = get_environment_name_from_full_name(self.full_env_name) diff --git a/scripts/build_env/tests/sd/test_process_sd_artifact.py b/scripts/build_env/tests/sd/test_process_sd_artifact.py index 9023c9697..30ded125c 100644 --- a/scripts/build_env/tests/sd/test_process_sd_artifact.py +++ b/scripts/build_env/tests/sd/test_process_sd_artifact.py @@ -1,8 +1,9 @@ import os +from pathlib import Path +from unittest.mock import patch import pytest -from unittest.mock import patch -from ruamel.yaml import YAML +from envgenehelper.env_helper import Environment from scripts.build_env.tests.base_test import BaseTest from test_sd_helpers import do_prerequisites, assert_sd_contents, load_test_pipeline_sd_data @@ -12,10 +13,8 @@ os.environ['CI_PROJECT_DIR'] = "temporary" from process_sd import handle_sd -from envgenehelper import * -from envgenehelper.env_helper import Environment +from envgenehelper import SD_FILE_NAME, logger, openJson -yaml = YAML() TEST_CASES_POSITIVE = [ "TC-001-098", @@ -33,22 +32,20 @@ "replace": [] } -SD = "sd.yaml" - FEATURE_TEST_DIR = "test_handle_sd" class TestSdProcessArtifact(BaseTest): - env_dir = "" - cluster_dir = "" - full_env_name = "cluster-01/env-01" def setup_method(self): + self.env_name = "env-01" + self.cluster = "cluster-01" + self.full_env_name = f"{self.cluster}/{self.env_name}" + self.set_ci_project_dir(FEATURE_TEST_DIR) self.test_data_dir = self.test_data_dir / FEATURE_TEST_DIR self.output_dir = self.output_dir / FEATURE_TEST_DIR - self.env_name = get_environment_name_from_full_name(self.full_env_name) - self.cluster = get_cluster_name_from_full_name(self.full_env_name) + os.environ["FULL_ENV_NAME"] = self.full_env_name os.environ["ENV_NAME"] = self.env_name os.environ["CLUSTER_NAME"] = self.cluster @@ -57,7 +54,7 @@ def setup_method(self): @patch("process_sd.download_sd_by_appver") def test_sd_positive(self, mock_download_sd, test_case_name): env = Environment(str(Path(self.output_dir, test_case_name)), self.cluster, self.env_name) - do_prerequisites(SD, self.test_data_dir, self.output_dir, test_case_name, env, test_suits_map) + do_prerequisites(SD_FILE_NAME, self.test_data_dir, self.output_dir, test_case_name, env, test_suits_map) logger.info(f"======TEST HANDLE_SD_ARTIFACT_POSITIVE: {test_case_name}======") logger.info(f"Starting SD test:" f"\n\tTest case: {test_case_name}") @@ -80,7 +77,7 @@ def test_sd_positive(self, mock_download_sd, test_case_name): def test_sd_negative(self, mock_download_sd, test_case_name, expected_exception): env = Environment(str(Path(self.output_dir, test_case_name)), self.cluster, self.env_name) - do_prerequisites(SD, self.test_data_dir, self.output_dir, test_case_name, env, test_suits_map) + do_prerequisites(SD_FILE_NAME, self.test_data_dir, self.output_dir, test_case_name, env, test_suits_map) logger.info(f"======TEST HANDLE_SD_ARTIFACT_NEGATIVE: {test_case_name}======") logger.info(f"Starting SD test:" f"\n\tTest case: {test_case_name}") diff --git a/scripts/build_env/tests/sd/test_process_sd_local.py b/scripts/build_env/tests/sd/test_process_sd_local.py index d6a4a8082..4fa855bba 100644 --- a/scripts/build_env/tests/sd/test_process_sd_local.py +++ b/scripts/build_env/tests/sd/test_process_sd_local.py @@ -1,8 +1,11 @@ import os +from pathlib import Path import pytest -from ruamel.yaml import YAML +from envgenehelper import logger, SD_FILE_NAME +from envgenehelper.env_helper import Environment +from scripts.build_env.tests.base_test import BaseTest from test_sd_helpers import do_prerequisites, assert_sd_contents, load_test_pipeline_sd_data os.environ['ENVIRONMENT_NAME'] = "temporary" @@ -10,10 +13,7 @@ os.environ['CI_PROJECT_DIR'] = "temporary" from process_sd import handle_sd -from envgenehelper import * -from envgenehelper.env_helper import Environment -yaml = YAML() TEST_CASES_POSITIVE = [ "TC-001-002", "TC-001-004", @@ -34,23 +34,36 @@ "replace": ["TC-001-008", "TC-001-006"] } -TEST_SD_DIR = Path(getAbsPath("../../test_data/test_handle_sd")) -OUTPUT_DIR = getAbsPath("../../tmp/test_handle_sd") -SD = "sd.yaml" +FEATURE_TEST_DIR = "test_handle_sd" + + +class TestSdProcessArtifact(BaseTest): + def setup_method(self): + self.env_name = "env-01" + self.cluster = "cluster-01" + self.full_env_name = f"{self.cluster}/{self.env_name}" + + self.set_ci_project_dir(FEATURE_TEST_DIR) + self.test_data_dir = self.test_data_dir / FEATURE_TEST_DIR + self.output_dir = self.output_dir / FEATURE_TEST_DIR + + os.environ["FULL_ENV_NAME"] = self.full_env_name + os.environ["ENV_NAME"] = self.env_name + os.environ["CLUSTER_NAME"] = self.cluster + @pytest.mark.parametrize("test_case_name", TEST_CASES_POSITIVE) + def test_sd_positive(self, test_case_name): + env = Environment(str(Path(self.output_dir, test_case_name)), "cluster-01", "env-01") + do_prerequisites(SD_FILE_NAME, self.test_data_dir, self.output_dir, test_case_name, env, test_suits_map) + logger.info(f"======TEST HANDLE_SD_LOCAL_POSITIVE: {test_case_name}======") + logger.info(f"Starting SD test:" + f"\n\tTest case: {test_case_name}") -@pytest.mark.parametrize("test_case_name", TEST_CASES_POSITIVE) -def test_sd_positive(test_case_name): - env = Environment(str(Path(OUTPUT_DIR, test_case_name)), "cluster-01", "env-01") - do_prerequisites(SD, TEST_SD_DIR, OUTPUT_DIR, test_case_name, env, test_suits_map) - logger.info(f"======TEST HANDLE_SD_LOCAL_POSITIVE: {test_case_name}======") - logger.info(f"Starting SD test:" - f"\n\tTest case: {test_case_name}") + sd_data, sd_source_type, sd_version, sd_delta, sd_merge_mode = load_test_pipeline_sd_data(self.test_data_dir, + test_case_name) - sd_data, sd_source_type, sd_version, sd_delta, sd_merge_mode = load_test_pipeline_sd_data(TEST_SD_DIR, test_case_name) - - handle_sd(env, sd_source_type, sd_version, sd_data, sd_delta, sd_merge_mode) - actual_dir = os.path.join(env.env_path, "Inventory", "solution-descriptor") + handle_sd(env, sd_source_type, sd_version, sd_data, sd_delta, sd_merge_mode) + actual_dir = os.path.join(env.env_path, "Inventory", "solution-descriptor") - assert_sd_contents(TEST_SD_DIR, OUTPUT_DIR, test_case_name, actual_dir, test_suits_map) - logger.info(f"=====SUCCESS - {test_case_name}======") + assert_sd_contents(self.test_data_dir, self.output_dir, test_case_name, actual_dir, test_suits_map) + logger.info(f"=====SUCCESS - {test_case_name}======") diff --git a/scripts/build_env/tests/sd/test_sd_helpers.py b/scripts/build_env/tests/sd/test_sd_helpers.py index c90ea8f35..b97405fdc 100644 --- a/scripts/build_env/tests/sd/test_sd_helpers.py +++ b/scripts/build_env/tests/sd/test_sd_helpers.py @@ -3,6 +3,7 @@ from envgenehelper import openYaml, writeYamlToFile from envgenehelper.test_helpers import TestHelpers + def load_test_pipeline_sd_data(test_sd_dir, test_case_name): file_path = Path(test_sd_dir, test_case_name, f"{test_case_name}.yaml") test_data = openYaml(file_path) @@ -27,6 +28,7 @@ def do_prerequisites(sd, test_sd_dir, output_dir, test_case_name, env, test_suit elif test_case_name in test_suits_map["extended"]: writeYamlToFile(target_sd_dir.joinpath(sd), openYaml(pr_dir.joinpath("extended").joinpath(sd))) + def assert_sd_contents(test_sd_dir, output_dir, test_case_name, actual_dir, test_suits_map): er_dir = test_sd_dir.joinpath("ER") diff --git a/test_data/test_handle_sd/ER/basic_not_first/delta_sd.yaml b/test_data/test_handle_sd/ER/basic_not_first/delta_sd.yaml deleted file mode 100644 index 9283f3e83..000000000 --- a/test_data/test_handle_sd/ER/basic_not_first/delta_sd.yaml +++ /dev/null @@ -1,5 +0,0 @@ -applications: - - version: postgres-services:1.32.6 - deployPostfix: postgresql - - version: postgres:1.32.7 - deployPostfix: postgresql-dbaas diff --git a/test_data/test_handle_sd/ER/exclude/delta_sd.yaml b/test_data/test_handle_sd/ER/exclude/delta_sd.yaml deleted file mode 100644 index 59938d6ec..000000000 --- a/test_data/test_handle_sd/ER/exclude/delta_sd.yaml +++ /dev/null @@ -1,5 +0,0 @@ -applications: - - version: postgres-services:1.32.6 - deployPostfix: postgresql - - version: postgres:1.32.6 - deployPostfix: postgresql-dbaas diff --git a/test_data/test_handle_sd/ER/extended/delta_sd.yaml b/test_data/test_handle_sd/ER/extended/delta_sd.yaml deleted file mode 100644 index 9fd14d0dd..000000000 --- a/test_data/test_handle_sd/ER/extended/delta_sd.yaml +++ /dev/null @@ -1,19 +0,0 @@ -version: 2.1 -type: solutionDeploy -deployMode: composite -applications: - - version: postgres:1.32.6 - deployPostfix: postgresql - - version: mysql:8.0.34 - deployPostfix: mysql-dbaas - - version: mongodb:7.0.3 - deployPostfix: mongo-db -deployGraph: - - chunkName: wave1 - apps: - - postgres:postgresql - - mysql:mysql-dbaas - - mongodb:mongo-db - - chunkName: wave2 - apps: - - redis:redis-cache