Feature/partial effective set generation

build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/FileSystemUtils.java

@@ -68,9 +68,6 @@ public void createEffectiveSetFolder(Optional<SolutionBomDTO> solutionDescriptor
 
     private void createEffectiveSetTwo(Optional<SolutionBomDTO> solutionDescriptor) throws IOException {
         File file = getFileFromGivenPath(data.getOutputDir());
-        if (file.exists()) {
-            FileUtils.forceDelete(file);
-        }
         file.mkdir();
         Path pipelinePath = getFileFromGivenPath(data.getOutputDir(), "pipeline").toPath();
         Files.createDirectories(pipelinePath);
@@ -92,13 +89,9 @@ private void createEffectiveSetTwo(Optional<SolutionBomDTO> solutionDescriptor)
                 }));
     }
 
-    private void createEffectiveSetOne(List<SBApplicationDTO> applicationDTOList) throws IOException {
+    private void createEffectiveSetOne(List<SBApplicationDTO> applicationDTOList) {
         File file = getFileFromGivenPath(data.getOutputDir());
-        if (file.exists()) {
-            FileUtils.forceDelete(file);
-        }
         file.mkdir();
-
         applicationDTOList
                 .forEach(app -> {
                     Path appPath = getFileFromGivenPath(data.getOutputDir(), app.getNamespace(), app.getAppName()).toPath();

build_effective_set_generator/scripts/effective_set_entrypoint.py

@@ -0,0 +1,164 @@
+import subprocess
+from os import getenv
+
+from envgenehelper.business_helper import get_environment_name_from_full_name, get_current_env_dir_from_env_vars
+from envgenehelper.file_helper import delete_dir, deleteFileIfExists, delete_dir_if_exists
+from envgenehelper.json_helper import openJson
+from envgenehelper.logger import logger
+from envgenehelper.yaml_helper import writeYamlToFile, openYaml
+from envgenehelper.effective_set_helper import resolve_es_generation_mode, GenerationMode, PartialMergeMode, \
+    resolve_partial_merge_mode, ES_MAPPING_FILE, ESGenerationContext
+from envgenehelper.sd_helper import get_sd_dir, DELTA_SD_FILE_NAME, SD_FILE_NAME
+
+from handle_effective_set_config import handle_effective_set_config
+
+
+def effective_set_entrypoint():
+    full_env_name = getenv("FULL_ENV_NAME")
+    effective_set_dir = get_current_env_dir_from_env_vars() / "effective-set"
+    sd_path = get_sd_dir().joinpath(SD_FILE_NAME)
+    delta_sd_path = get_sd_dir().joinpath(DELTA_SD_FILE_NAME)
+
+    if resolve_es_generation_mode() == GenerationMode.PARTIAL:
+        if not delta_sd_path.exists() and sd_path.exists():
+            raise ValueError(
+                f"[Partial effective set generation] impossible without delta sd {delta_sd_path} and full sd {sd_path}")
+        # first run
+        if not effective_set_dir.exists():
+            _run_full_generation(effective_set_dir, full_env_name, sd_path)
+
+        elif resolve_partial_merge_mode() == PartialMergeMode.REVERSE:
+            _run_reverse_merge(effective_set_dir, full_env_name, delta_sd_path)
+        else:
+            _run_forward_merge(effective_set_dir, full_env_name, delta_sd_path)
+    else:
+        _run_full_generation(effective_set_dir, full_env_name, sd_path)
+
+    # do not commit delta sd to repo
+    deleteFileIfExists(delta_sd_path)
+
+
+def _run_full_generation(effective_set_dir, full_env_name, sd_path):
+    cmd = _build_cli_cmd(effective_set_dir, full_env_name, sd_path)
+    delete_dir(effective_set_dir)
+    subprocess.run(["sh", cmd], check=True)
+
+
+def _run_forward_merge(effective_set_dir, full_env_name, delta_sd_path):
+    cmd = _build_cli_cmd(effective_set_dir, full_env_name, delta_sd_path)
+
+    delta_sd = openYaml(delta_sd_path)
+    apps = delta_sd.get("applications", [])
+
+    deploy_postfixes = {
+        app.get("deployPostfix")
+        for app in apps
+        if app.get("deployPostfix")
+    }
+
+    for ns in deploy_postfixes:
+        delete_dir_if_exists(effective_set_dir / ESGenerationContext.CLEANUP.value / ns)
+
+    for app in apps:
+        app_name = app.get("version", "").split(":")[0]
+        ns = app.get("deployPostfix")
+        delete_dir_if_exists(effective_set_dir / ESGenerationContext.RUNTIME.value / ns / app_name)
+        delete_dir_if_exists(effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ns / app_name)
+
+    delete_dir_if_exists(effective_set_dir / ESGenerationContext.TOPOLOGY.value)
+    delete_dir_if_exists(effective_set_dir / ESGenerationContext.PIPELINE.value)
+
+    cleanup_mapping_path = effective_set_dir / ESGenerationContext.CLEANUP.value / ES_MAPPING_FILE
+    runtime_mapping_path = effective_set_dir / ESGenerationContext.RUNTIME.value / ES_MAPPING_FILE
+    deployment_mapping_path = effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ES_MAPPING_FILE
+
+    cleanup_mapping = openYaml(cleanup_mapping_path, allow_default=True)
+    runtime_mapping = openYaml(runtime_mapping_path, allow_default=True)
+    deployment_mapping = openYaml(deployment_mapping_path, allow_default=True)
+
+    subprocess.run(["sh", cmd], check=True)
+
+    new_cleanup_mapping = openYaml(cleanup_mapping_path, allow_default=True)
+    new_runtime_mapping = openYaml(runtime_mapping_path, allow_default=True)
+    new_deployment_mapping = openYaml(deployment_mapping_path, allow_default=True)
+
+    cleanup_mapping.update(new_cleanup_mapping)
+    runtime_mapping.update(new_runtime_mapping)
+    deployment_mapping.update(new_deployment_mapping)
+
+    writeYamlToFile(cleanup_mapping_path, cleanup_mapping)
+    writeYamlToFile(runtime_mapping_path, runtime_mapping)
+    writeYamlToFile(deployment_mapping_path, deployment_mapping)
+
+
+def _run_reverse_merge(effective_set_dir, full_env_name, delta_sd_path):
+    apps = openYaml(delta_sd_path).get("applications", [])
+
+    mapping_paths = [
+        effective_set_dir / ESGenerationContext.CLEANUP.value / ES_MAPPING_FILE,
+        effective_set_dir / ESGenerationContext.RUNTIME.value / ES_MAPPING_FILE,
+        effective_set_dir / ESGenerationContext.DEPLOYMENT.value / ES_MAPPING_FILE,
+    ]
+    ns = get_environment_name_from_full_name(full_env_name)
+
+    for app in apps:
+        app_name = app.get("version", "").split(":")[0]
+        dp = app.get("deployPostfix")
+
+        runtime_dp = effective_set_dir / ESGenerationContext.RUNTIME.value / dp
+        deployment_dp = effective_set_dir / ESGenerationContext.DEPLOYMENT.value / dp
+        cleanup_dp = effective_set_dir / ESGenerationContext.CLEANUP.value / dp
+
+        delete_dir(runtime_dp / app_name)
+        delete_dir(deployment_dp / app_name)
+
+        if any(runtime_dp.iterdir()):
+            continue
+
+        delete_dir(runtime_dp)
+        delete_dir(deployment_dp)
+        delete_dir(cleanup_dp)
+
+        for path in mapping_paths:
+            if not path.exists():
+                logger.warning(f"Mapping file not found, skipping: {path}")
+                continue
+            mapping = openYaml(path, allow_default=True) or {}
+            if ns not in mapping:
+                logger.warning(f"Namespace '{ns}' not found in mapping file {path}, skipping removing by key: {ns}")
+                continue
+            mapping.pop(ns)
+            writeYamlToFile(path, mapping)
+
+
+def _build_cli_cmd(effective_set_dir, full_env_name, sd_path):
+    cmd = [
+        "/module/scripts/utils/run_effective_set_cli.sh",
+        f"--env-id={full_env_name}",
+        "--envs-path=$CI_PROJECT_DIR/environments",
+        f"--output={effective_set_dir}",
+    ]
+
+    if sd_path.is_file():
+        cmd.extend([
+            "--registries=${CI_PROJECT_DIR}/configuration/registry.yml",
+            "--sboms-path=$sboms_path",
+            "--sd-path=$sd_path",
+        ])
+
+    effective_set_config = getenv("EFFECTIVE_SET_CONFIG")
+    if effective_set_config:
+        handle_effective_set_config(effective_set_config)
+        effective_set_output = openJson("/tmp/effective_set_output.json")
+        extra_args = effective_set_output.get("extra_args") or []
+        cmd.extend(extra_args)
+
+    deployment_id = getenv("DEPLOYMENT_SESSION_ID")
+    if deployment_id:
+        cmd.append(f"--extra_params=DEPLOYMENT_SESSION_ID={deployment_id}")
+
+    custom_params = getenv("CUSTOM_PARAMS")
+    if custom_params:
+        cmd.append(f"--custom-params={custom_params}")
+
+    return cmd

build_pipegene/scripts/effective_set_job.py

@@ -1,93 +1,39 @@
 import json
+import os
 from os import getenv, environ
 from pathlib import Path
 
+from envgenehelper import logger, copy_path
 from gcip import WhenStatement, Need
 
-from envgenehelper import logger, get_sboms_dir
-from envgenehelper import cleanup_targets
 from pipeline_helper import job_instance
 
 
 def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluster_name, params):
     logger.info(f'Prepare generate-effective-set job for {full_env_name}')
-    logger.info(f'Cleanup_targets: {cleanup_targets}')
-
-    app_reg_defs_job = params["APP_REG_DEFS_JOB"]
-    artifact_app_defs_path = params["APP_DEFS_PATH"]
-    artifact_reg_defs_path = params["REG_DEFS_PATH"]
-    sd_version = params["SD_VERSION"]
-    sd_data = params["SD_DATA"]
-    deployment_id = params["DEPLOYMENT_SESSION_ID"]
-    effective_set_config = params["EFFECTIVE_SET_CONFIG"]
-    if "CUSTOM_PARAMS" in params:
-        custom_params = params["CUSTOM_PARAMS"]
 
-    is_local_app_def = artifact_app_defs_path and artifact_reg_defs_path and app_reg_defs_job
+    effective_set_config = params["EFFECTIVE_SET_CONFIG"]
+    if effective_set_config:
+        logger.info(f"EFFECTIVE_SET_CONFIG: {effective_set_config}")
+        effective_set_config_dict = json.loads(effective_set_config)
+        validate_topology_context_mode(effective_set_config_dict, full_env_name, params)
 
-    base_dir = getenv('CI_PROJECT_DIR')   
+    app_reg_defs_job = params.get("APP_REG_DEFS_JOB")
+    is_local_app_def = init_local_app_defs_from_artifact(full_env_name, app_reg_defs_job, params)
 
-    sd_path = Path(f'{base_dir}/environments/{full_env_name}/Inventory/solution-descriptor/sd.yaml')
-    # TODO it is necessary to remove unnecessary calls, leave only script calls in such jobs! bad for gsf delivery
     script = [
-        #Overriding sd_path to pick the correct value for CI_PROJECT_DIR
-        f'base_env_path="$CI_PROJECT_DIR/environments/{full_env_name}";',
-        'app_defs_path="$base_env_path/AppDefs";',
-        'reg_defs_path="$base_env_path/RegDefs";',
-        'sboms_path="$CI_PROJECT_DIR/sboms";',
-        'sd_path="$base_env_path/Inventory/solution-descriptor/sd.yaml";',
         # cert handling for java
         'mkdir -p ${CI_PROJECT_DIR}/configuration/certs/',
         'if [ -f /default_cert.pem ]; then cp /default_cert.pem "${CI_PROJECT_DIR}/configuration/certs/"; fi',
         'for cert in "${CI_PROJECT_DIR}/configuration/certs/*" ; do [ -f "$cert" ] && keytool -import -trustcacerts -alias "$(basename "$cert")" -file "$cert" -keystore /etc/ssl/certs/keystore.jks -storepass changeit -noprompt; done',
-        'python3 /module/scripts/main.py decrypt_cred_files',
-        f'[ -n "$APP_REG_DEFS_JOB" ] && [ -n "$APP_DEFS_PATH" ] && mkdir -p $app_defs_path && cp -rf {artifact_app_defs_path}/* $app_defs_path',
-        f'[ -n "$APP_REG_DEFS_JOB" ] && [ -n "$REG_DEFS_PATH" ] && mkdir -p $reg_defs_path && cp -fr {artifact_reg_defs_path}/* $reg_defs_path',
-        'python3 /module/scripts/main.py validate_creds',
-        'python3 /module/scripts/sboms_retention_policy.py'
-    ]
 
-    cmdb_cli_cmd_call = [
-        f"/module/scripts/utils/entrypoint.sh --env-id={full_env_name}",
-        "--envs-path=$CI_PROJECT_DIR/environments",
-        f"--output=$CI_PROJECT_DIR/environments/{full_env_name}/effective-set"
+        'python3 /module/scripts/crypt_manager.py decrypt_cred_files',
+        'python3 /module/scripts/crypt_manager.py validate_creds',
+        'python3 /module/scripts/sboms_retention_policy.py',
+        'python3 /module/scripts/effective_set_entrypoint.py',
+        'python3 /module/scripts/crypt_manager.py encrypt_cred_files',
     ]
 
-    effective_set_config_dict = {}
-    if effective_set_config:
-        effective_set_config_dict = json.loads(effective_set_config)
-
-    effective_set_version = effective_set_config_dict.get("version") or "v2.0"
-    full_sd_exists = sd_path.is_file()
-    sd_data = bool(sd_data) or bool(sd_version)
-
-    if not (full_sd_exists and sd_data) and effective_set_version.lower() == "v1.0":
-        raise ValueError("Feature generation effective set for pipeline and topology context is not supported for v1.0")
-
-    if full_sd_exists or sd_data:
-        cmdb_cli_cmd_call.extend([
-            "--registries=${CI_PROJECT_DIR}/configuration/registry.yml",
-            "--sboms-path=$sboms_path",
-            "--sd-path=$sd_path",
-        ])
-
-    logger.info(f'Prepare generate_effective_set job for {full_env_name}.')
-    if effective_set_config:
-        logger.info(f"EFFECTIVE_SET_CONFIG: {effective_set_config}")
-        script.extend([
-            f"python3 /module/scripts/handle_effective_set_config.py --effective-set-config '{effective_set_config}'",
-            'extra_args=$(jq -r \'.extra_args // [] | join(" ")\' /tmp/effective_set_output.json)',
-        ])
-        cmdb_cli_cmd_call.extend(["$extra_args"])
-    if deployment_id:
-        cmdb_cli_cmd_call.extend([f"--extra_params=DEPLOYMENT_SESSION_ID={deployment_id}"])
-
-    if custom_params:
-        logger.info(f"custom_params : {custom_params}")
-        cmdb_cli_cmd_call.extend([f"--custom-params='{custom_params}'"])
-    script.append(" ".join(cmdb_cli_cmd_call))
-    script.append('python3 /module/scripts/main.py encrypt_cred_files')
-
     generate_effective_set_params = {
         "name": f'generate_effective_set.{full_env_name}',
         "image": '${effective_set_generator_image}',
@@ -101,7 +47,7 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste
         "ENV_NAME": env_name,
         "INSTANCES_DIR": "${CI_PROJECT_DIR}/environments",
         "effective_set_generator_image": "$effective_set_generator_image",
-        "EXCLUDE_CLEANUP_TARGETS": " ".join(cleanup_targets)
+        "FULL_ENV_NAME": full_env_name,
     }
 
     needs = []
@@ -113,9 +59,11 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste
         environ['CI_PIPELINE_ID'] = '0000000'
         needs.append(Need(job=app_reg_defs_job, pipeline=real_ci_pipe_id, artifacts=True))
         environ['CI_PIPELINE_ID'] = real_ci_pipe_id
+
     generate_effective_set_job = job_instance(params=generate_effective_set_params, needs=needs,
-                                                              vars=generate_effective_set_vars)
+                                              vars=generate_effective_set_vars)
 
+    effective_set_config_dict = {}
     effective_set_expiry = effective_set_config_dict.get("effective_set_expiry") or "1 hour"
     logger.info(f"effective set expiry value '{effective_set_expiry}'")
     generate_effective_set_job.artifacts.expire_in = effective_set_expiry
@@ -124,3 +72,29 @@ def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluste
     pipeline.add_children(generate_effective_set_job)
 
     return generate_effective_set_job
+
+
+def validate_topology_context_mode(effective_set_config_dict, full_env_name, params):
+    effective_set_version = effective_set_config_dict.get("version") or "v2.0"
+    sd_input = bool(params["SD_DATA"]) or bool(params["SD_VERSION"])
+    full_sd_path = Path(
+        f'{getenv('CI_PROJECT_DIR')}/environments/{full_env_name}/Inventory/solution-descriptor/sd.yaml')
+    any_sd = full_sd_path.exists() and sd_input
+    # effective set generation in version 1.0 does not support no sd mode
+    if not any_sd and effective_set_version.lower() == "v1.0":
+        raise ValueError("Feature generation effective set for pipeline and topology context is not supported for v1.0")
+
+    if not any_sd:
+        logger.info("No-SD Mode: no SD present, only topology and pipeline contexts are generated; "
+                    "deployment, runtime, and cleanup are skipped, SBOMs are not requested")
+
+
+def init_local_app_defs_from_artifact(full_env_name, app_reg_defs_job, params):
+    work_dir = os.path.join(getenv('CI_PROJECT_DIR'), "environments", full_env_name)
+    artifact_app_defs_path = params.get("APP_DEFS_PATH")
+    artifact_reg_defs_path = params.get("REG_DEFS_PATH")
+    is_local_app_def = artifact_app_defs_path and artifact_reg_defs_path and app_reg_defs_job
+    if is_local_app_def:
+        copy_path(artifact_app_defs_path, os.path.join(work_dir, "AppDefs"))
+        copy_path(artifact_reg_defs_path, os.path.join(work_dir, "RegDefs"))
+    return is_local_app_def

build_pipegene/scripts/process_sd_job.py

@@ -1,14 +1,12 @@
-from os import getenv
-
+from envgenehelper import logger
 from gcip import WhenStatement
 
-from envgenehelper import logger
 from pipeline_helper import job_instance
 
 
 def prepare_process_sd(pipeline, full_env, environment_name, cluster_name):
     logger.info(f'Prepare process_sd job for {full_env}')
-    
+
     script = [
         f'base_env_path="$CI_PROJECT_DIR/environments/{full_env}";',
         'app_defs_path="$base_env_path/AppDefs";',
@@ -28,12 +26,12 @@ def prepare_process_sd(pipeline, full_env, environment_name, cluster_name):
     process_sd_set_vars = {
         "CLUSTER_NAME": cluster_name,
         "ENVIRONMENT_NAME": environment_name,
-        "ENV_NAME": environment_name,
         "INSTANCES_DIR": "${CI_PROJECT_DIR}/environments",
+        "FULL_ENV_NAME": full_env,
     }
 
     process_sd_job = job_instance(params=process_sd_set_params, vars=process_sd_set_vars)
-    process_sd_job.artifacts.when = WhenStatement.ALWAYS    
+    process_sd_job.artifacts.when = WhenStatement.ALWAYS
     pipeline.add_children(process_sd_job)
-    
-    return process_sd_job
+
+    return process_sd_job

python/envgene/envgenehelper/__init__.py

@@ -7,7 +7,7 @@
 from .collections_helper import *
 from .logger import logger
 from .creds_helper import *
-from .sd_merge_helper import *
+from .sd_helper import *
 from .yaml_validator import checkByWhiteList, checkByBlackList, checkSchemaValidationFailed, getSchemaValidationErrorMessage
 from .crypt import decrypt_file, encrypt_file, decrypt_all_cred_files_for_env, encrypt_all_cred_files_for_env, is_encrypted
-from .constants import cleanup_targets
+from .constants import cleanup_targets