diff --git a/.github/actions/build-effective-set/action.yml b/.github/actions/build-effective-set/action.yml index a7b65eb65..322e319cb 100644 --- a/.github/actions/build-effective-set/action.yml +++ b/.github/actions/build-effective-set/action.yml @@ -18,7 +18,7 @@ inputs: dockerfile-path: description: 'Path to Dockerfile' required: false - default: './build_effective_set_generator/Dockerfile' + default: './build_effective_set_generator/build/Dockerfile' git-user: description: 'Git username for build args' required: false @@ -52,6 +52,8 @@ runs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + ref: ${{ inputs.version != '' && format('v{0}', inputs.version) || github.ref }} - name: Download JAR artifact uses: actions/download-artifact@v4 diff --git a/.github/actions/build-envgene/action.yml b/.github/actions/build-envgene/action.yml index a7024255f..c826802c0 100644 --- a/.github/actions/build-envgene/action.yml +++ b/.github/actions/build-envgene/action.yml @@ -49,6 +49,8 @@ runs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + ref: ${{ inputs.version != '' && format('v{0}', inputs.version) || github.ref }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 diff --git a/.github/actions/build-gsf-discovery/action.yml b/.github/actions/build-gsf-discovery/action.yml index 396a96557..71389c81f 100644 --- a/.github/actions/build-gsf-discovery/action.yml +++ b/.github/actions/build-gsf-discovery/action.yml @@ -53,8 +53,18 @@ runs: - name: Checkout repository uses: actions/checkout@v4 with: - token: ${{ inputs.github-token }} - fetch-depth: 0 + ref: ${{ inputs.version != '' && format('v{0}', inputs.version) || github.ref }} + + - name: Update history.log + shell: bash + run: | + COOKIECUTTER_DIR="gsf_packages/envgene_discovery_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}" + HISTORY_FILE="gsf_packages/envgene_discovery_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/history.log" + + # Run Python script to update history.log + python3 ./.github/scripts/update_history.py \ + "$COOKIECUTTER_DIR" \ + "$HISTORY_FILE" - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -73,33 +83,6 @@ runs: id: tags uses: ./.github/actions/generate-docker-tags - - name: Update history.log - shell: bash - run: | - COOKIECUTTER_DIR="gsf_packages/envgene_discovery_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}" - HISTORY_FILE="gsf_packages/envgene_discovery_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/history.log" - - # Run Python script to update history.log - python3 ./.github/scripts/update_history.py \ - "$COOKIECUTTER_DIR" \ - "$HISTORY_FILE" - - - name: Commit and push changes - shell: bash - run: | - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git remote set-url origin https://x-access-token:${{ inputs.github-token }}@github.com/${GITHUB_REPOSITORY}.git - - BRANCH_NAME=${GITHUB_REF#refs/heads/} - BRANCH_NAME=${BRANCH_NAME#refs/tags/} - - git add -A - if ! git diff --quiet --cached; then - git commit -m "chore: Update GSF Discovery instance package files [skip ci]" - git push origin HEAD:"${BRANCH_NAME}" || echo "Failed to push changes" - fi - - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v5 diff --git a/.github/actions/build-gsf-instance/action.yml b/.github/actions/build-gsf-instance/action.yml index 80529f428..dd99cfe87 100644 --- a/.github/actions/build-gsf-instance/action.yml +++ b/.github/actions/build-gsf-instance/action.yml @@ -53,25 +53,7 @@ runs: - name: Checkout repository uses: actions/checkout@v4 with: - token: ${{ inputs.github-token }} - fetch-depth: 0 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ inputs.github-token }} - - - name: Generate Docker tags - id: tags - uses: ./.github/actions/generate-docker-tags + ref: ${{ inputs.version != '' && format('v{0}', inputs.version) || github.ref }} - name: Copy config.schema.json to git_hooks shell: bash @@ -91,21 +73,22 @@ runs: "$COOKIECUTTER_DIR" \ "$HISTORY_FILE" - - name: Commit and push changes - shell: bash - run: | - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git remote set-url origin https://x-access-token:${{ inputs.github-token }}@github.com/${GITHUB_REPOSITORY}.git + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 - BRANCH_NAME=${GITHUB_REF#refs/heads/} - BRANCH_NAME=${BRANCH_NAME#refs/tags/} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ inputs.github-token }} - git add -A - if ! git diff --quiet --cached; then - git commit -m "chore: Update GSF EnvGene instance package files [skip ci]" - git push origin HEAD:"${BRANCH_NAME}" || echo "Failed to push changes" - fi + - name: Generate Docker tags + id: tags + uses: ./.github/actions/generate-docker-tags - name: Extract metadata (tags, labels) for Docker id: meta diff --git a/.github/actions/build-pipegene/action.yml b/.github/actions/build-pipegene/action.yml index 6f8e7172d..aa9478ae2 100644 --- a/.github/actions/build-pipegene/action.yml +++ b/.github/actions/build-pipegene/action.yml @@ -52,6 +52,8 @@ runs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + ref: ${{ inputs.version != '' && format('v{0}', inputs.version) || github.ref }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 diff --git a/.github/actions/build-pipeline/action.yml b/.github/actions/build-pipeline/action.yml index 485a4117e..e8d728535 100644 --- a/.github/actions/build-pipeline/action.yml +++ b/.github/actions/build-pipeline/action.yml @@ -35,6 +35,8 @@ runs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + ref: ${{ inputs.version != '' && format('v{0}', inputs.version) || github.ref }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 970c63452..4aa24289d 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -30,7 +30,7 @@ runs: cp dependencies/sources.list /etc/apt/sources.list fi - pip install --upgrade pip setuptools wheel + pip install --upgrade pip "setuptools<82" wheel pip install --no-cache-dir -r dependencies/tests_requirements.txt if [ -f python/build_modules.sh ]; then @@ -44,6 +44,11 @@ runs: mv sops-v3.9.0.linux.amd64 /usr/local/bin/sops chmod +x /usr/local/bin/sops + - name: Configure test environment + shell: bash + run: | + echo "PYTHONPATH=${GITHUB_WORKSPACE}" >> "${GITHUB_ENV}" + - name: ENVGENE HELPER test shell: bash run: | @@ -52,6 +57,14 @@ runs: cd ../../.. mv junit.xml junit_envgenehelper.xml + - name: PIPEGENE test + shell: bash + run: | + cd build_pipegene/scripts + pytest --capture=no -W ignore::DeprecationWarning --junitxml=../../junit.xml + cd ../.. + mv junit.xml junit_pipegene.xml + - name: ARTIFACT SEARCHER test shell: bash run: | @@ -88,10 +101,18 @@ runs: cd ../.. mv junit.xml junit_cred_rotation.xml + - name: SBOMS RETENTION POLICY test + shell: bash + run: | + cd build_effective_set_generator/scripts + pytest --capture=no -W ignore::DeprecationWarning --junitxml=../../junit.xml + cd ../.. + mv junit.xml junit_sbom_retention.xml + - name: Merge test results shell: bash run: | - junitparser merge junit_build_env.xml junit_envgenehelper.xml junit.xml + junitparser merge junit_*.xml junit.xml - name: Upload test results if: always() diff --git a/.github/workflows/docker_publish_release.yml b/.github/workflows/docker_publish_release.yml index 730a95bb5..e44e752ae 100644 --- a/.github/workflows/docker_publish_release.yml +++ b/.github/workflows/docker_publish_release.yml @@ -86,7 +86,7 @@ jobs: fetch-depth: 0 token: ${{ steps.app-token.outputs.token || secrets.GITHUB_TOKEN }} - - name: Update Docker Image Tags in Pipeline + - name: Add the Envgene version to pipeline and GSF run: | # Paths to files TARGET_FILE="github_workflows/instance-repo-pipeline/.github/workflows/Envgene.yml" @@ -110,46 +110,18 @@ jobs: # Update version in GSF Discovery package.yaml sed -i "/version:/c\\version: ${NEW_TAG}" "$GSF_DISCOVERY_PACKAGE_FILE" - CHANGES_MADE=false - if ! git diff --quiet "$TARGET_FILE"; then - echo "Updated Envgene.yml with docker tags from job outputs" - CHANGES_MADE=true - fi - - if ! git diff --quiet "$ENVGENE_COOKIECUTTER_JSON_FILE"; then - echo "Updated cookiecutter.json with envgene_version" - CHANGES_MADE=true - fi - - if ! git diff --quiet "$GSF_INSTANCE_PACKAGE_FILE"; then - echo "Updated package.yaml with version" - CHANGES_MADE=true - fi - - if ! git diff --quiet "$GSF_DISCOVERY_PACKAGE_FILE"; then - echo "Updated package.yaml with version" - CHANGES_MADE=true - fi - - if [ "$CHANGES_MADE" = false ]; then - echo "No changes to commit - no images were built or tags were already up to date" - fi + git diff - git diff "$TARGET_FILE" || true - git diff "$ENVGENE_COOKIECUTTER_JSON_FILE" || true - git diff "$GSF_INSTANCE_PACKAGE_FILE" || true - git diff "$GSF_DISCOVERY_PACKAGE_FILE" || true - - if [ "$CHANGES_MADE" = true ]; then - git config --global user.name "qubership-actions[bot]" - git config --global user.email "qubership-actions[bot]@users.noreply.github.com" - git add "$TARGET_FILE" "$ENVGENE_COOKIECUTTER_JSON_FILE" "$GSF_INSTANCE_PACKAGE_FILE" "$GSF_DISCOVERY_PACKAGE_FILE" + git config --global user.name "qubership-actions[bot]" + git config --global user.email "qubership-actions[bot]@users.noreply.github.com" + git add . + if ! git diff --quiet --cached; then git commit -m "chore: Update docker image tags and envgene_version for branch ${{ github.ref_name }} [skip ci]" - git push origin HEAD:${{ github.ref_name }} || echo "Failed to push changes" else - echo "No changes to commit - skipping git operations" + echo "No changes to commit - working tree clean" fi + tag: needs: [update_image_tags] uses: netcracker/qubership-workflow-hub/.github/workflows/tag-creator.yml@v1.0.3 @@ -214,7 +186,7 @@ jobs: java-version: 17 upload-artifact: true artifact-id: effective_set_jar - ref: ${{ github.ref }} + ref: v${{ github.event.inputs.version }} secrets: maven-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/link-checker.yaml b/.github/workflows/link-checker.yaml index b11e70963..2b2e8a60b 100644 --- a/.github/workflows/link-checker.yaml +++ b/.github/workflows/link-checker.yaml @@ -41,10 +41,13 @@ --max-retries 8 --accept 100..=103,200..=299,429 --cookie-jar cookies.json - --exclude-all-private + --include 'https?://.*' --max-concurrency 4 --cache --cache-exclude-status '429, 500..502' --max-cache-age 1d + --include file + --include-fragments + --root-dir $GITHUB_WORKSPACE format: markdown fail: true diff --git a/.github/workflows/sonar-check.yml b/.github/workflows/sonar-check.yml new file mode 100644 index 000000000..f90382ed1 --- /dev/null +++ b/.github/workflows/sonar-check.yml @@ -0,0 +1,28 @@ +name: Sonar Check + +on: + workflow_dispatch: {} + push: + branches: [main] + +jobs: + mvn-build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build java + uses: netcracker/qubership-workflow-hub/actions/maven-snapshot-deploy@main + with: + java-version: 17 + pom-file: build_effective_set_generator/pom.xml + # Default target-store is "central"; the action then adds -Pcentral and GPG runs. Sonar only needs -Pgithub. + target-store: github + maven-command: > + --batch-mode clean verify + org.sonarsource.scanner.maven:sonar-maven-plugin:${{ vars.SONAR_PLUGIN_VERSION }}:sonar + -Dsonar.projectKey=${{ vars.SONAR_PROJECT_KEY }} + -Dsonar.organization=${{ vars.SONAR_ORGANIZATION }} + -Dsonar.host.url=${{ vars.SONAR_HOST_URL }} + maven-token: ${{ secrets.GITHUB_TOKEN }} + sonar-token: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 82f61d8db..566f689ec 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ __pycache__ .vscode .DS_Store pyrightconfig.json +junit* diff --git a/.qubership/grand-report.json b/.qubership/grand-report.json index f216aab40..9da304c8c 100644 --- a/.qubership/grand-report.json +++ b/.qubership/grand-report.json @@ -1,5 +1,8 @@ { "exclusions" : [ { + "t-hash" : "00000000", + "f-hash" : "519368c1daa626a147f2a8f67e9ceec568378a9b094fd3f6ecd625faa2a1d564" + }, { "t-hash" : "823c4eb3e895adc925a755d89cea1c6c46954c999d23604e0091788b75496159", "f-hash" : "680348e409a5a7a1ccae4a38eea6315f78eab8d683e599d12bf03bc1405b1a75" }, { @@ -8,5 +11,29 @@ }, { "t-hash" : "244f28ce3685167745ad3a7f1760fd4483bbbb3fd150b9087b95442d4d6fd905", "f-hash" : "f81b4f0ec55a8c76b2006d0d8fb8cf55ba73f8f24db12cd9e737c06e50f8010c" + }, { + "t-hash" : "f454bc9163706b52dc68c37847db48967e15b2f18a80c1d85f4387e9970cd299", + "f-hash" : "4a097b2592a091a5e9c9898fb01b957287f62d282e37abdc3f64f5397ddf7790" + }, { + "t-hash" : "f454bc9163706b52dc68c37847db48967e15b2f18a80c1d85f4387e9970cd299", + "f-hash" : "1b81a1c8c26e93bbbc06c67a3aeebffaa9fb32e4adc12ea43c546bd8ba6b0db2" + }, { + "t-hash" : "f454bc9163706b52dc68c37847db48967e15b2f18a80c1d85f4387e9970cd299", + "f-hash" : "266ceec1c40da5d4664b1fd8b680fb11ecab4bc58c28ca4b83c95c2cfc87da42" + }, { + "t-hash" : "f454bc9163706b52dc68c37847db48967e15b2f18a80c1d85f4387e9970cd299", + "f-hash" : "12358bb89894a3b900c43b8fbaf976baaecfd1151976ade3a7528cdc47728c52" + }, { + "t-hash" : "f454bc9163706b52dc68c37847db48967e15b2f18a80c1d85f4387e9970cd299", + "f-hash" : "81364271645a7f08f26b6d896bc26221e14ae46b670d66a2990bd3b805c18da3" + }, { + "t-hash" : "f454bc9163706b52dc68c37847db48967e15b2f18a80c1d85f4387e9970cd299", + "f-hash" : "633b0a67c34213aeba550c218df84e75f1839993ec7fef1c8f6c4f9d924f507f" + }, { + "t-hash" : "f454bc9163706b52dc68c37847db48967e15b2f18a80c1d85f4387e9970cd299", + "f-hash" : "490876a57b0af0ebbb422281ec7b178483e8ea4462281533b2a34abbf3725222" + }, { + "t-hash" : "f454bc9163706b52dc68c37847db48967e15b2f18a80c1d85f4387e9970cd299", + "f-hash" : "fe68378b7ff1c467cce4d2579b704fcac1b1cd7bae254f7b5225586ca59233d0" } ] } diff --git a/AGENTS.md b/AGENTS.md index e2426c03f..45df5629b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,6 +75,63 @@ Content... - Remove special characters - Example: `### Step 1: Install Tools` → `#step-1-install-tools` +#### Dashes + +**CRITICAL: Always use a regular hyphen-minus (`-`) as a dash in prose. Never use em dashes (`—`) or en dashes (`–`).** + +❌ **INCORRECT:** + +```markdown +EnvGene searches these locations — from bottom to top — and uses the first match. +``` + +✅ **CORRECT:** + +```markdown +EnvGene searches these locations - from bottom to top - and uses the first match. +``` + +**Why:** Em dashes are a typographic convention that varies by locale and style guide. A plain hyphen-minus is universally readable, renders consistently across all Markdown renderers, and avoids accidental character encoding issues. + +--- + +#### Callouts (Notes, Warnings, Tips) + +**CRITICAL: Always use GitHub-flavored Markdown native callout syntax, not bold-text workarounds.** + +Available types: `NOTE`, `TIP`, `IMPORTANT`, `WARNING`, `CAUTION`. + +❌ **INCORRECT:** + +```markdown +> **Note:** EnvGene also supports dot-notation keys. + +> **Warning:** This will overwrite existing values. +``` + +✅ **CORRECT:** + +```markdown +> [!NOTE] +> EnvGene also supports dot-notation keys. + +> [!WARNING] +> This will overwrite existing values. + +> [!TIP] +> Use cluster-wide scope to avoid repetition across environments. + +> [!IMPORTANT] +> The `name` field must exactly match the filename without the extension. + +> [!CAUTION] +> Setting `mergeEnvSpecificResourceProfiles: false` replaces the template override entirely. +``` + +**Why:** Native callouts render with icons and colour highlighting on GitHub and other renderers; bold-text variants are plain blockquotes. + +--- + #### Tables **CRITICAL: All Markdown tables MUST have vertically aligned pipe characters (`|`).** @@ -108,10 +165,10 @@ Content... **How to achieve alignment:** -1. **Keep cell content concise** — Long text makes alignment difficult -2. **Simplify when possible** — Remove examples from cells if they make text too long -3. **Uniform width per column** — Each cell in a column should have the same width (add trailing spaces) -4. **Don't add spaces endlessly** — If alignment fails repeatedly, the problem is content length, not spacing +1. **Keep cell content concise** - Long text makes alignment difficult +2. **Simplify when possible** - Remove examples from cells if they make text too long +3. **Uniform width per column** - Each cell in a column should have the same width (add trailing spaces) +4. **Don't add spaces endlessly** - If alignment fails repeatedly, the problem is content length, not spacing ##### Common Mistake @@ -151,9 +208,71 @@ Content... --- +## Object Examples in Documentation + +### Source of Truth for Object Schemas + +**CRITICAL: Never invent object structures. Always derive examples from authoritative sources.** + +The two authoritative sources are: + +- **`docs/envgene-objects.md`** - human-readable descriptions, field explanations, and canonical examples for all EnvGene objects +- **`schemas/`** - JSON Schema files that define required fields, allowed values, and types + +#### Rules + +1. **Before writing any YAML/JSON example** for an EnvGene object, read the corresponding entry in `docs/envgene-objects.md` AND the matching schema file under `schemas/`. +2. **Validate every example against the schema**: all fields marked `"required"` in the schema must be present; no fields may be included that do not exist in the schema (unless `additionalProperties: true`). +3. **Do not guess**: if an object is not described in `docs/envgene-objects.md` and has no schema file, write explicitly: + + > No schema or description found for this object in `docs/envgene-objects.md` or `schemas/`. Cannot provide a validated example. + +4. **Do not add fictional fields** such as `type:` or `applications:` to objects that have no such fields in their schema. +5. **Use real field names**: cross-check field names and allowed enum values against the schema. Do not invent field names based on intuition. + +#### How much of the object to show + +In tutorials and how-to guides, show only the **relevant part** of the object, not the full structure. Use `# ...` comments to signal omitted fields so the reader knows the snippet is intentionally incomplete. + +- **Reference docs** → show the full object. +- **Tutorials / how-to guides** → show only the fields being explained; collapse the rest with `# ...`. + +This keeps examples focused on the concept being taught and avoids becoming outdated when unrelated fields change. + +#### ❌ INCORRECT - invented fields and unnecessary noise + +```yaml +# Namespace template - WRONG: invented fields, full object shown in tutorial context +name: "{{ current_env.name }}-bss" +type: namespace # does not exist in namespace.schema.json +applications: # does not exist in namespace.schema.json + - name: "Cloud-BSS" +credentialsId: "" +isServerSideMerge: false +cleanInstallApprovalRequired: false +mergeDeployParametersAndE2EParameters: false +deployParameterSets: + - "bss" +``` + +#### ✅ CORRECT - focused snippet, validated field names, omissions annotated + +```yaml +# Namespace template - only the relevant section is shown +--- +name: "{{ current_env.environmentName }}-bss" +# ... other required fields (see schemas/namespace.schema.json) ... +profile: + name: dev-bss-override + baseline: dev +# ... deployParameterSets, e2eParameters, etc. ... +``` + +--- + ## Documentation Structure (Diátaxis Framework) -This repository follows the [Diátaxis documentation framework](https://diataxis.fr/). +This repository follows the [Diátaxis documentation framework](https://github.com/evildmp/diataxis-documentation-framework). ### Documentation Types diff --git a/Makefile b/Makefile index 4270b9941..e933632ff 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ build-%: up-%: $(compose) up -d $* - @if [ -f devtools/$*/up.sh ]; then $(compose) exec $* sh /workspace/devtools/$*/up.sh; fi + @if [ -f devtools/$*/up.sh ]; then $(compose) exec $* bash /workspace/devtools/$*/up.sh; fi bash-%: $(compose) exec $* bash @@ -20,7 +20,7 @@ rm-%: $(compose) rm $* run-%: - @if [ -f devtools/$*/run.sh ]; then $(compose) exec $* sh /workspace/devtools/$*/run.sh; \ + @if [ -f devtools/$*/run.sh ]; then $(compose) exec $* bash /workspace/devtools/$*/run.sh; \ else echo "No run script for $*"; fi edit: diff --git a/README.md b/README.md index 420a362bd..b5aa19700 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,10 @@ - [Basic Usage](#basic-usage) - [📚 Documentation](#-documentation) - [Getting Started](#getting-started) + - [Tutorials](#tutorials) - [Core Concepts](#core-concepts) - [How-To Guides](#how-to-guides) + - [Migrations](#migrations) - [Advanced Features](#advanced-features) - [Examples \& Samples](#examples--samples) - [Development](#development) @@ -99,7 +101,7 @@ EnvGene simplifies Environment management by providing: ``` > [!NOTE] For special instructions on the GitHub pipeline, see [GH_ADDITIONAL_PARAMS docs](/docs/instance-pipeline-parameters.md) - > and the [pipeline description](/github_workflows/instance-repo-pipeline/.github/docs/README.md) + > and the [pipeline description](/github_workflows/instance-repo-pipeline/.github/README.md) After the pipeline finishes, the Environment configuration will be generated and committed to your instance repository: @@ -112,6 +114,11 @@ After the pipeline finishes, the Environment configuration will be generated and - [**Quick Start Guide**](#-quick-start) - Create your first Environment +### Tutorials + +- [**Understanding the Effective Set**](/docs/tutorials/effective-set.md) - Trace how parameters from Tenant, Cloud, Namespace, Application, and SBOM sources are merged into the final Effective Set; learn to read traceability comments and debug wrong values +- [**Managing Resource Profiles**](/docs/tutorials/resource-profiles.md) - End-to-end walkthrough: from Baseline to Template Override to Environment-Specific Override, including `template_override`, `overrides-parent`, and result verification + ### Core Concepts - [**EnvGene Objects**](/docs/envgene-objects.md) - What are EnvGene objects and how they work @@ -136,11 +143,20 @@ After the pipeline finishes, the Environment configuration will be generated and - [**Override Template Parameters**](/docs/how-to/environment-specific-parameters.md) - Override template parameters for specific environments - [**Configure Resource Profiles**](/docs/how-to/configure-resource-profiles.md) - Configure performance parameters for different environment types +**Effective Set:** + +- [**Generate an Effective Set**](/docs/how-to/generate-effective-set.md) - Trigger Effective Set generation from a Solution Descriptor artifact and template version + **Advanced Configuration:** - [**Configure Namespace Names for Sites**](/docs/how-to/configure-ns-names-for-sites.md) - Site-specific namespace naming +- [**Filter Namespaces in Template Descriptor**](/docs/how-to/filter-ns-in-template-descriptor.md) - Generate Environments with selected namespaces only - [**Credential Encryption**](/docs/how-to/credential-encryption.md) - Secure credential storage and rotation + +### Migrations + - [**Migrate to Dot-Notated Parameters**](/docs/how-to/dot-notated-parameter-migration.md) - Parameter format migration +- [**Migrate SBOM Storage to Per-Application Layout**](/docs/how-to/sbom-storage-migration.md) - Transition to per-application SBOM directory layout when upgrading EnvGene ### Advanced Features @@ -151,12 +167,15 @@ After the pipeline finishes, the Environment configuration will be generated and - [**Environment Instance Generation**](/docs/features/environment-instance-generation.md) - Generate Environment Instances from templates and inventories - [**Credential Rotation**](/docs/features/cred-rotation.md) - Automate [Credential](/docs/envgene-objects.md#credential) rotation - [**Namespace Render Filter**](/docs/features/namespace-render-filtering.md) - Render only selected [Namespaces](/docs/envgene-objects.md#namespace) +- [**Namespace Filtering in Template Descriptor**](/docs/features/namespace-filtering-in-template-descriptor.md) - Filter namespaces during Template Descriptor rendering - [**System Certificate Configuration**](/docs/features/system-certificate.md) - Auto-config system certs for internal registries or TLS services - [**Template Override**](/docs/features/template-override.md) - Use a base Environment template and override parts as needed - [**Automatic Environment Name Derivation**](/docs/features/auto-env-name-derivation.md) - Auto-detect Environment name from folder structure -- [**Template Inheritance**](/docs/features/template-inheritance.md) - Advanced Environment template patterns +- [**Template Composition**](/docs/features/template-composition.md) - Advanced Environment template patterns - [**Blue-Green Deployment**](/docs/features/blue-green-deployment.md) - BG domains, state management, and `bg_manage` pipeline job - [**Resource Profiles**](/docs/features/resource-profile.md) - Baselines and overrides for performance parameters +- [**SBOM**](/docs/features/sbom.md) - CycloneDX-based artifact and parameter exchange for EnvGene +- [**SBOM Retention**](/docs/features/sbom-retention.md) - Automatic cleanup of cached SBOM files to manage repository size ### Examples & Samples diff --git a/base_modules/scripts/decrypt_fernet.py b/base_modules/scripts/decrypt_fernet.py index 86aa61192..9d6d60f1c 100644 --- a/base_modules/scripts/decrypt_fernet.py +++ b/base_modules/scripts/decrypt_fernet.py @@ -16,7 +16,6 @@ def cmdb_prepare(): @click.option('--secret_key', '-s', 'secret_key', required=True, help="Set secret_key for encrypt cred files") def decrypt_file(secret_key, file_path): - ''' {getenv('CI_PROJECT_DIR')}/ansible/inventory/group_vars/{getenv('env_name')}/appdeployer_cmdb/Tenants/{getenv('tenant_name')}/Credentials''' logger.debug('Try to read %s file', file_path) with open(file_path, mode="r", encoding="utf-8") as sensitive: sensitive_data = safe_load(sensitive) diff --git a/base_modules/scripts/update_ca_certs.sh b/base_modules/scripts/update_ca_certs.sh deleted file mode 100755 index e6e42d240..000000000 --- a/base_modules/scripts/update_ca_certs.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -CA_FILE="$1" - -function getLinuxDisto { - if [[ -f /etc/os-release ]]; then - # freedesktop.org and systemd - # shellcheck disable=SC1091 - . /etc/os-release - DIST=$NAME - elif type lsb_release >/dev/null 2>&1; then - # linuxbase.org - DIST=$(lsb_release -si) - elif [[ -f /etc/lsb-release ]]; then - # For some versions of Debian/Ubuntu without lsb_release command - # shellcheck disable=SC1091 - . /etc/lsb-release - DIST=$DISTRIB_ID - elif [[ -f /etc/debian_version ]]; then - # Older Debian/Ubuntu/etc. - DIST=Debian - else - # Fall back to uname, e.g. "Linux ", also works for BSD, etc. - DIST=$(uname -s) - fi - # convert to lowercase - DIST="$(tr '[:upper:]' '[:lower:]' <<<"$DIST")" -} - -function updateCertificates { - if [[ -e "${CA_FILE}" && -n "${CA_FILE}" ]]; then - getLinuxDisto - echo "Linux Distribution identified as: $DIST" - if [[ "${DIST}" == *"debian"* || "${DIST}" == *"ubuntu"* ]]; then - cp "${CA_FILE}" /usr/local/share/ca-certificates/ca.crt - update-ca-certificates --fresh >/dev/null - elif [[ "${DIST}" == *"centos"* ]]; then - cp "${CA_FILE}" /etc/pki/ca-trust/source/anchors/ca.crt - update-ca-trust - elif [[ "${DIST}" == *"alpine"* ]]; then - cat "${CA_FILE}" >>/etc/ssl/certs/ca-certificates.crt - echo "certs from $CA_FILE added to trusted root" - fi - else - echo "CA file ${CA_FILE} not found or empty" - exit 1 - fi -} - -updateCertificates diff --git a/build_effective_set_generator/.gitignore b/build_effective_set_generator/.gitignore index a572cf5b7..25ff83d14 100644 --- a/build_effective_set_generator/.gitignore +++ b/build_effective_set_generator/.gitignore @@ -36,6 +36,9 @@ **/build.gradle **/settings.gradle **/build +# Exception: build directory for Docker build files (must be after **/build rule) +!build/ +!build/** # CMake cmake-build-debug/ diff --git a/build_effective_set_generator/Dockerfile b/build_effective_set_generator/Dockerfile deleted file mode 100644 index e9cd485cd..000000000 --- a/build_effective_set_generator/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -FROM registry.access.redhat.com/ubi8/ubi-minimal:8.1 AS builder - -ARG JAVA_PACKAGE=java-17-openjdk-headless -ARG RUN_JAVA_VERSION=1.3.8 - -# hadolint ignore=DL3041 -RUN microdnf install -y curl ca-certificates ${JAVA_PACKAGE} && \ - microdnf clean all && \ - mkdir -p /deployments && \ - curl -s https://repo1.maven.org/maven2/io/fabric8/run-java-sh/${RUN_JAVA_VERSION}/run-java-sh-${RUN_JAVA_VERSION}-sh.sh -o /deployments/run-java.sh && \ - chmod 540 /deployments/run-java.sh && \ - echo "securerandom.source=file:/dev/urandom" >> /etc/alternatives/jre/lib/security/java.security - -COPY build_effective_set_generator/effective-set-generator/target/*.jar /deployments/app.jar - -FROM registry.access.redhat.com/ubi8/ubi-minimal:8.1 - -ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' - -COPY --from=builder --chown=1001:root /deployments /deployments -COPY --from=builder /etc/alternatives/jre/lib/security/java.security /etc/alternatives/jre/lib/security/java.security - -## NOTE: This script requires Python and will fail unless Python is added in the future. -COPY scripts/utils /module/scripts/utils - -RUN chmod g+rwX /deployments - -# Ensure sane permissions on copied tree without findutils -RUN chmod -R u=rwX,go=rX /deployments && \ - chmod 540 /deployments/run-java.sh - -USER 1001 - -ENTRYPOINT [ "/deployments/run-java.sh" ] diff --git a/build_effective_set_generator/build/Dockerfile b/build_effective_set_generator/build/Dockerfile new file mode 100644 index 000000000..68d89a8da --- /dev/null +++ b/build_effective_set_generator/build/Dockerfile @@ -0,0 +1,71 @@ +# ═══════════════════════════════════════════════════════════════════════════════════ +# Stage 1: Build +# ═══════════════════════════════════════════════════════════════════════════════════ +FROM ghcr.io/netcracker/qubership-envgene-base-modules:1.0.0 AS build + +ARG RUN_JAVA_VERSION=1.3.8 + +# hadolint ignore=DL3018 +RUN apk add --no-cache \ + libffi-dev \ + openssl-dev \ + python3-dev \ + cargo \ + rust \ + build-base + +COPY build_effective_set_generator/build/pip.conf /etc/pip.conf +COPY build_effective_set_generator/build/requirements.txt /build/requirements.txt +COPY python /python + +RUN pip install --no-cache-dir \ + --no-binary cffi \ + --no-binary cryptography \ + -r /build/requirements.txt + +RUN pip install --no-cache-dir \ + /python/integration \ + /python/jschon-sort \ + /python/envgene \ + /python/artifact-searcher + +RUN mkdir -p /deployments && \ + curl -sSL https://repo1.maven.org/maven2/io/fabric8/run-java-sh/${RUN_JAVA_VERSION}/run-java-sh-${RUN_JAVA_VERSION}-sh.sh \ + -o /deployments/run-java.sh + +COPY build_effective_set_generator/effective-set-generator/target/*.jar /deployments/app.jar + + +# ═══════════════════════════════════════════════════════════════════════════════════ +# Stage 2: Runtime +# ═══════════════════════════════════════════════════════════════════════════════════ +FROM ghcr.io/netcracker/qubership-envgene-base-modules:1.0.0 AS runtime + +ARG JAVA_PACKAGE=openjdk17-jre-headless + +ENV LANG='en_US.UTF-8' \ + LANGUAGE='en_US:en' + +# Install ONLY runtime dependencies (NO dev packages) +# hadolint ignore=DL3018 +RUN apk add --no-cache \ + libgcc \ + ${JAVA_PACKAGE} + +COPY --from=build /python /python +COPY --from=build /module/venv /module/venv +COPY --from=build /deployments /deployments + +COPY scripts/build_env/ /build_env/scripts/build_env/ +COPY build_effective_set_generator/scripts/ /module/scripts/ +COPY scripts/utils/ /module/scripts/utils/ + +# Permissions +RUN chown -R ci:ci /module /deployments && \ + chmod +x /module/scripts/*.sh && \ + chmod 644 /module/scripts/*.py 2>/dev/null || true && \ + chmod 540 /deployments/run-java.sh && \ + chmod g+rwX /deployments + +USER ci:ci +WORKDIR /module diff --git a/build_effective_set_generator/build/pip.conf b/build_effective_set_generator/build/pip.conf new file mode 100644 index 000000000..8cfdd31ec --- /dev/null +++ b/build_effective_set_generator/build/pip.conf @@ -0,0 +1,4 @@ +[global] +index-url=https://pypi.org/simple +extra-index-url=https://example.com/pypi/simple +trusted-host=pypi.org diff --git a/build_effective_set_generator/build/requirements.txt b/build_effective_set_generator/build/requirements.txt new file mode 100644 index 000000000..52a6f65b7 --- /dev/null +++ b/build_effective_set_generator/build/requirements.txt @@ -0,0 +1,32 @@ +# Base module requirements (essential only) +PyGithub==1.55 +certifi==2022.6.15 + +boto3==1.39.4 +botocore==1.39.4 +gcip==3.0.2 +jmespath==1.0.1 +packaging==23.2 +python-dateutil==2.8.2 +PyYAML==6.0.2 +s3transfer==0.13.1 +setuptools-git-versioning==1.13.5 +six==1.16.0 +toml==0.10.2 +urllib3==2.0.7 +lxml==4.9.3 +ruamel.yaml==0.18.5 +ruamel.yaml.clib==0.2.8 +ruyaml==0.91.0 +jschon==0.11.0 +jsonschema==4.24.1 +diagrams==0.24.1 +graphviz==0.20.3 +attrs==23.2.0 +referencing==0.33.0 +rpds-py==0.17.1 +jsonschema-specifications==2023.12.1 +cryptography==41.0.3 +click==8.1.7 +deepmerge==2.0 +flatten_dict==0.4.2 \ No newline at end of file diff --git a/build_effective_set_generator/build/sources.list b/build_effective_set_generator/build/sources.list new file mode 100644 index 000000000..91b969a08 --- /dev/null +++ b/build_effective_set_generator/build/sources.list @@ -0,0 +1,2 @@ +https://dl-cdn.alpinelinux.org/alpine/v3.20/main +https://dl-cdn.alpinelinux.org/alpine/v3.20/community diff --git a/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/pojo/parameterset/CustomParameterDTO.java b/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/pojo/parameterset/CustomParameterDTO.java new file mode 100644 index 000000000..e8de43779 --- /dev/null +++ b/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/pojo/parameterset/CustomParameterDTO.java @@ -0,0 +1,46 @@ +/* + * Copyright 2024-2025 NetCracker Technology Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.qubership.cloud.devops.commons.pojo.parameterset; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.jackson.Jacksonized; +import org.qubership.cloud.devops.commons.utils.Parameter; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +@Data +@Builder +@Jacksonized +@AllArgsConstructor +@NoArgsConstructor +public class CustomParameterDTO { + + private Map deployParams = Collections.emptyMap(); + private Map technicalParams = Collections.emptyMap(); + + public Map getAllParams() { + Map params = new HashMap<>(); + params.putAll(deployParams); + params.putAll(technicalParams); + return params; + } +} \ No newline at end of file diff --git a/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/HelmNameNormalizer.java b/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/HelmNameNormalizer.java index 74daedc55..8046c1e58 100644 --- a/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/HelmNameNormalizer.java +++ b/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/HelmNameNormalizer.java @@ -44,7 +44,7 @@ private static String normalizeNameForHelm(String name, int limit) { } // Convert binary mask to decimal - int decimalMask = Integer.parseInt(mask.toString(), 2); + long decimalMask = Long.parseLong(mask.toString(), 2); // Encode decimal mask in base-36 using custom symbols List base36Digits = numberToBase(decimalMask, 36); @@ -62,14 +62,14 @@ private static String normalizeNameForHelm(String name, int limit) { return finalName; } - private static List numberToBase(int number, int base) { + private static List numberToBase(long number, int base) { List digits = new ArrayList<>(); if (number == 0) { digits.add(0); return digits; } while (number > 0) { - digits.add(0, number % base); + digits.add(0, (int)(number % base)); number /= base; } return digits; diff --git a/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/ParameterUtils.java b/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/ParameterUtils.java index 84ecaae8f..399ab3ace 100644 --- a/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/ParameterUtils.java +++ b/build_effective_set_generator/commons/src/main/java/org/qubership/cloud/devops/commons/utils/ParameterUtils.java @@ -18,6 +18,7 @@ import lombok.experimental.UtilityClass; import org.apache.commons.collections4.MapUtils; +import org.qubership.cloud.devops.commons.pojo.parameterset.CustomParameterDTO; import java.util.*; @@ -117,6 +118,23 @@ public static void splitBgDomainParams(Map bgDomainMap, bgDomainParamsMap.put(CONTROLLER_NAMESPACE, controller); bgDomainSecureMap.put(CONTROLLER_NAMESPACE, Map.of(USERNAME, userName, PASSWORD, password)); } -} + public static void prepareCustomParams(CustomParameterDTO customParameterDTO, + Map deployParams, Map technicalParams) { + updateParameter(customParameterDTO.getAllParams(), deployParams); + updateParameter(customParameterDTO.getAllParams(), technicalParams); + } + + private static void updateParameter(Map customParams, Map params) { + for (Map.Entry entry : customParams.entrySet()) { + String key = entry.getKey(); + Parameter customParam = entry.getValue(); + if (params.containsKey(key)) { + Parameter deployParam = params.get(key); + customParam.setValue(deployParam.getValue()); + } + params.remove(key); + } + } +} diff --git a/build_effective_set_generator/effective-set-generator/pom.xml b/build_effective_set_generator/effective-set-generator/pom.xml index cf787ce87..83773a1bb 100644 --- a/build_effective_set_generator/effective-set-generator/pom.xml +++ b/build_effective_set_generator/effective-set-generator/pom.xml @@ -45,6 +45,11 @@ + + org.yaml + snakeyaml + 2.3 + io.quarkus quarkus-picocli diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/CmdbCli.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/CmdbCli.java index 7727a7fef..42b7dd5e8 100644 --- a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/CmdbCli.java +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/CmdbCli.java @@ -16,6 +16,10 @@ package org.qubership.cloud.devops.cli; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.SneakyThrows; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.qubership.cloud.devops.cli.parser.CliParameterParser; @@ -26,12 +30,12 @@ import lombok.extern.slf4j.Slf4j; import picocli.CommandLine; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; +import java.util.*; import java.util.concurrent.Callable; import static org.qubership.cloud.devops.cli.exceptions.constants.ExceptionMessage.EFFECTIVE_SET_FAILED; @@ -49,6 +53,7 @@ public class CmdbCli implements Callable { @Inject FileDataRepositoryImpl fileDataRepository; + @Inject CliParameterParser parser; @@ -67,7 +72,7 @@ public Integer call() { return 0; } catch (Exception e) { logError(String.format(EFFECTIVE_SET_FAILED, e.getMessage())); - logDebug(String.format("Stack trace: %s", ExceptionUtils.getStackTrace(e))); + logError(String.format("Stack trace: %s", ExceptionUtils.getStackTrace(e))); return 1; } } @@ -102,6 +107,7 @@ private void validateVersionDependentParams(EffectiveSetVersion version) { } } + @SneakyThrows private void setSharedData() { EffectiveSetVersion effectiveVersion = EffectiveSetVersion.fromString(envParams.version); sharedData.setEffectiveSetVersion(effectiveVersion); @@ -114,9 +120,40 @@ private void setSharedData() { sharedData.setOutputDir(envParams.outputDir); sharedData.setPcsspPaths(envParams.pcssp != null ? List.of(envParams.pcssp) : new ArrayList<>()); sharedData.setAppChartValidation(envParams.appChartValidation); + prepareCustomParameters(getCustomParams(envParams.customParams)); populateDeploymentSessionId(envParams.extraParams); } + @SneakyThrows + private String getCustomParams(String customParams) { + if (StringUtils.isNotEmpty(customParams) && + customParams.startsWith("@")) { + String fileName = customParams.substring(1); + Path projectRoot = Paths.get(System.getProperty("user.dir")); + Path path = projectRoot + .resolve("src/test/resources") + .resolve(fileName) + .normalize(); + customParams = Files.readString(path); + } + return customParams; + } + + private void prepareCustomParameters(String customParams) throws JsonProcessingException { + if (StringUtils.isEmpty(customParams)) { + return; + } + ObjectMapper mapper = new ObjectMapper(); + Map> map = mapper.readValue(customParams, new TypeReference<>() { + }); + if (map.containsKey("deployment")) { + sharedData.setCustomDeployParamMap(map.get("deployment")); + } + if (map.containsKey("runtime")) { + sharedData.setCustomRuntimeParamMap(map.get("runtime")); + } + } + private void populateDeploymentSessionId(String[] extraParams) { if (extraParams != null) { @@ -161,6 +198,9 @@ static class EnvCommandSpace { @CommandLine.Option(names = {"-acv", "--app_chart_validation"}, description = "App chart validation parameter on sbom", arity = "1") boolean appChartValidation = true; + @CommandLine.Option(names = {"-cp", "--custom-params"}, description = "Custom Parameters") + String customParams; + } } diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/parser/CliParameterParser.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/parser/CliParameterParser.java index d8da095b2..19c15529f 100644 --- a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/parser/CliParameterParser.java +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/parser/CliParameterParser.java @@ -41,10 +41,13 @@ import org.qubership.cloud.devops.commons.pojo.credentials.model.Credential; import org.qubership.cloud.devops.commons.pojo.credentials.model.UsernamePasswordCredentials; import org.qubership.cloud.devops.commons.pojo.namespaces.dto.NamespaceDTO; +import org.qubership.cloud.devops.commons.pojo.parameterset.CustomParameterDTO; import org.qubership.cloud.devops.commons.repository.interfaces.FileDataConverter; import org.qubership.cloud.devops.commons.utils.CredentialUtils; import org.qubership.cloud.devops.commons.utils.HelmNameNormalizer; +import org.qubership.cloud.devops.commons.utils.Parameter; import org.qubership.cloud.devops.commons.utils.ParameterUtils; +import org.qubership.cloud.devops.commons.utils.constant.ParametersConstants; import org.qubership.cloud.parameters.processor.dto.DeployerInputs; import org.qubership.cloud.parameters.processor.dto.ParameterBundle; import org.qubership.cloud.parameters.processor.service.ParametersCalculationServiceV1; @@ -96,19 +99,20 @@ public void generateEffectiveSet() throws IOException, IllegalArgumentException, processAndSaveParameters(inputData.getSolutionBomDTO(), tenantName, cloudName, namespaceDTOMap); } - private void processAndSaveParameters(Optional solutionDescriptor, String tenantName, String cloudName, Map namespaceDTOMap) throws IOException { + private void processAndSaveParameters(Optional solutionDescriptor, String tenantName, String cloudName, Map namespaceDTOMap) throws IOException { Map deployMappingFileData = new ConcurrentHashMap<>(); Map runtimeMappingFileData = new ConcurrentHashMap<>(); Map cleanupMappingFileData = new ConcurrentHashMap<>(); Map errorList = new ConcurrentHashMap<>(); Map k8TokenMap = new ConcurrentHashMap<>(); namespaceDTOMap.keySet().parallelStream().forEach(namespaceName -> { + String originalNamespace = inputData.getNamespaceDTOMap().get(namespaceName).getName(); String credentialsId = findDefaultCredentialsId(namespaceName); if (StringUtils.isNotEmpty(credentialsId)) { CredentialDTO credentialDTO = inputData.getCredentialDTOMap().get(credentialsId); if (credentialDTO != null) { SecretCredentialsDTO secCred = (SecretCredentialsDTO) credentialDTO.getData(); - k8TokenMap.put(namespaceName, secCred.getSecret()); + k8TokenMap.put(originalNamespace, secCred.getSecret()); } } }); @@ -148,7 +152,7 @@ private void processAndSaveParameters(Optional solutionDescripto }); if (EffectiveSetVersion.V2_0 == sharedData.getEffectiveSetVersion()) { generateE2EOutput(tenantName, cloudName, k8TokenMap); - if (solutionDescriptor.isPresent()) { + if (solutionDescriptor.isPresent()) { fileDataConverter.writeToFile(new TreeMap<>(deployMappingFileData), sharedData.getOutputDir(), "deployment", "mapping.yaml"); fileDataConverter.writeToFile(new TreeMap<>(runtimeMappingFileData), sharedData.getOutputDir(), "runtime", "mapping.yaml"); fileDataConverter.writeToFile(new TreeMap<>(cleanupMappingFileData), sharedData.getOutputDir(), "cleanup", "mapping.yaml"); @@ -180,17 +184,6 @@ private void generateE2EOutput(String tenantName, String cloudName, Map k8TokenMap) throws IOException { - ParameterBundle parameterBundle = parametersServiceV2.getCleanupParameterBundle(tenantName, cloudName, namespace, null, originalNamespace, k8TokenMap); - if (parameterBundle.getCleanupParameters() == null) { - parameterBundle.setCleanupParameters(new HashMap<>()); - } - if (parameterBundle.getCleanupSecureParameters() == null) { - parameterBundle.setCleanupSecureParameters(new HashMap<>()); - } - createCleanupFiles(parameterBundle, namespace); - } - private void processBgDomainParameters() { BgDomainEntityDTO bgDomainEntityDTO = inputData.getBgDomainEntityDTO(); if (bgDomainEntityDTO != null && bgDomainEntityDTO.getControllerNamespace().getCredentials() != null) { @@ -204,12 +197,6 @@ private void processBgDomainParameters() { } } - private void createCleanupFiles(ParameterBundle parameterBundle, String namespace) throws IOException { - String cleanupDir = String.format("%s/%s/%s", sharedData.getOutputDir(), "cleanup", namespace); - fileDataConverter.writeToFile(parameterBundle.getCleanupParameters(), cleanupDir, "parameters.yaml"); - fileDataConverter.writeToFile(parameterBundle.getCleanupSecureParameters(), cleanupDir, "credentials.yaml"); - } - private void createTopologyFiles(Map k8TokenMap) throws IOException { Map topologyParams = new TreeMap<>(); Map topologySecuredParams = new TreeMap<>(); @@ -287,37 +274,61 @@ public void generateOutput(String tenantName, String cloudName, String namespace String appVersion, String appFileRef, Map k8TokenMap) throws IOException { DeployerInputs deployerInputs = DeployerInputs.builder().appVersion(appVersion).appFileRef(appFileRef).build(); String originalNamespace = inputData.getNamespaceDTOMap().get(namespaceName).getName(); - String credentialsId = findDefaultCredentialsId(namespaceName); - if (StringUtils.isNotEmpty(credentialsId)) { - CredentialDTO credentialDTO = inputData.getCredentialDTOMap().get(credentialsId); - if (credentialDTO != null) { - SecretCredentialsDTO secCred = (SecretCredentialsDTO) credentialDTO.getData(); - k8TokenMap.put(originalNamespace, secCred.getSecret()); - } - } - ParameterBundle parameterBundle = null; + ParameterBundle parameterBundle; if (EffectiveSetVersion.V2_0 == sharedData.getEffectiveSetVersion()) { + CustomParameterDTO customParams = getCustomParameters(); parameterBundle = parametersServiceV2.getCliParameter(tenantName, cloudName, namespaceName, appName, deployerInputs, originalNamespace, - k8TokenMap); - generateCleanupOutput(tenantName, cloudName, namespaceName, originalNamespace, k8TokenMap); + k8TokenMap, + customParams); + ParameterBundle cleanupParameterBundle = parametersServiceV2.getCleanupParameterBundle(tenantName, cloudName, namespaceName, null, originalNamespace, k8TokenMap); + createCleanupParams(parameterBundle, cleanupParameterBundle); } else { parameterBundle = parametersServiceV1.getCliParameter(tenantName, cloudName, namespaceName, appName, deployerInputs, - originalNamespace, - k8TokenMap); + originalNamespace); } createFiles(namespaceName, appName, parameterBundle, originalNamespace); } + private CustomParameterDTO getCustomParameters() { + CustomParameterDTO parameterDTO = CustomParameterDTO.builder().build(); + Map deployParams = new HashMap<>(); + Map techParams = new HashMap<>(); + sharedData.getCustomDeployParamMap().forEach((key, value) -> { + deployParams.put(key, new Parameter(value, ParametersConstants.CUSTOM_PARAMS_ORIGIN, false)); + }); + sharedData.getCustomRuntimeParamMap().forEach((key, value) -> { + techParams.put(key, new Parameter(value, ParametersConstants.CUSTOM_PARAMS_ORIGIN, false)); + }); + parameterDTO.setDeployParams(deployParams); + parameterDTO.setTechnicalParams(techParams); + return parameterDTO; + } + + private void createCleanupParams(ParameterBundle parameterBundle, ParameterBundle cleanupParameterBundle) { + if (cleanupParameterBundle.getCleanupParameters() == null) { + cleanupParameterBundle.setCleanupParameters(new HashMap<>()); + } + if (cleanupParameterBundle.getCleanupSecureParameters() == null) { + cleanupParameterBundle.setCleanupSecureParameters(new HashMap<>()); + } + if (MapUtils.isNotEmpty(cleanupParameterBundle.getCleanupSecureParameters()) && + MapUtils.isNotEmpty(parameterBundle.getCustomTechParameters())) { + cleanupParameterBundle.getCleanupSecureParameters().putAll(parameterBundle.getCustomTechParameters()); + } + parameterBundle.setCleanupParameters(cleanupParameterBundle.getCleanupParameters()); + parameterBundle.setCleanupSecureParameters(cleanupParameterBundle.getCleanupSecureParameters()); + } + private String findDefaultCredentialsId(String namespace) { return !StringUtils.isEmpty(inputData.getNamespaceDTOMap().get(namespace).getCredentialsId()) ? inputData.getNamespaceDTOMap().get(namespace).getCredentialsId() : inputData.getCloudDTO().getDefaultCredentialsId(); @@ -335,6 +346,10 @@ private void createFiles(String namespaceName, String appName, ParameterBundle p String deploymentDir = String.format("%s/%s/%s/%s/%s", sharedData.getOutputDir(), "deployment", namespaceName, appName, "values"); String runtimeDir = String.format("%s/%s/%s/%s", sharedData.getOutputDir(), "runtime", namespaceName, appName); + String cleanupDir = String.format("%s/%s/%s", sharedData.getOutputDir(), "cleanup", namespaceName); + fileDataConverter.writeToFile(parameterBundle.getCleanupParameters(), cleanupDir, "parameters.yaml"); + fileDataConverter.writeToFile(parameterBundle.getCleanupSecureParameters(), cleanupDir, "credentials.yaml"); + //deployment fileDataConverter.writeToFile(parameterBundle.getDeployParams(), deploymentDir, "deployment-parameters.yaml"); if (StringUtils.isNotBlank(parameterBundle.getAppChartName())) { @@ -359,6 +374,7 @@ private void createFiles(String namespaceName, String appName, ParameterBundle p //runtime parameters fileDataConverter.writeToFile(parameterBundle.getConfigServerParams(), runtimeDir, "parameters.yaml"); fileDataConverter.writeToFile(parameterBundle.getSecuredConfigParams(), runtimeDir, "credentials.yaml"); + fileDataConverter.writeToFile(parameterBundle.getCustomDeployParameters(), deploymentDir, "custom-params.yaml"); } else { String appDirectory = String.format("%s/%s/%s", sharedData.getOutputDir(), namespaceName, appName); fileDataConverter.writeToFile(parameterBundle.getDeployParams(), appDirectory, "deployment-parameters.yaml"); diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/pojo/dto/shared/SharedData.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/pojo/dto/shared/SharedData.java index 72303bdae..46f22f750 100644 --- a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/pojo/dto/shared/SharedData.java +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/pojo/dto/shared/SharedData.java @@ -18,10 +18,13 @@ import jakarta.enterprise.context.ApplicationScoped; +import lombok.Builder; import lombok.Getter; import lombok.Setter; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; @Getter @@ -49,4 +52,9 @@ public class SharedData { private boolean appChartValidation; + @Builder.Default + private Map customDeployParamMap = Collections.emptyMap(); + @Builder.Default + private Map customRuntimeParamMap = Collections.emptyMap(); + } diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/repository/implementation/FileDataConverterImpl.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/repository/implementation/FileDataConverterImpl.java index 214b92d46..06007809e 100644 --- a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/repository/implementation/FileDataConverterImpl.java +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/repository/implementation/FileDataConverterImpl.java @@ -34,13 +34,14 @@ import org.yaml.snakeyaml.nodes.Node; import org.yaml.snakeyaml.nodes.Tag; import org.yaml.snakeyaml.representer.Representer; +import org.qubership.cloud.devops.cli.utils.yaml.AdaptiveYaml; import java.io.*; import java.util.Base64; import java.util.Map; import java.util.TreeMap; -import static org.qubership.cloud.devops.commons.utils.ConsoleLogger.logError; +import static org.qubership.cloud.devops.commons.utils.ConsoleLogger.*; @ApplicationScoped @@ -96,9 +97,16 @@ public T parseInputFile(TypeReference typeReference, File file) { @Override public void writeToFile(Map params, String... args) throws IOException { File file = fileSystemUtils.getFileFromGivenPath(args); + + boolean expand = params != null && !params.isEmpty() + && AdaptiveYaml.shouldExpand(params); + if (expand) { + logInfo("removing anchors and aliases for file: " + file.getAbsolutePath()); + } + try (BufferedWriter writer = new BufferedWriter(new FileWriter(file))) { if (params != null && !params.isEmpty()) { - getYamlObject().dump(params, writer); + getYamlObject(expand).dump(params, writer); } } } @@ -112,11 +120,14 @@ public Map getObjectMap(T inputObject) { } - private static Yaml getYamlObject() { + private static Yaml getYamlObject(boolean expand) { DumperOptions options = new DumperOptions(); options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); options.setDefaultScalarStyle(DumperOptions.ScalarStyle.PLAIN); options.setPrettyFlow(false); + if (expand) { + options.setDereferenceAliases(true); + } Representer representer = new Representer(options) { @Override protected Node representScalar(Tag tag, String value, DumperOptions.ScalarStyle style) { diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/repository/implementation/FileDataRepositoryImpl.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/repository/implementation/FileDataRepositoryImpl.java index f741b9133..0d6b37314 100644 --- a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/repository/implementation/FileDataRepositoryImpl.java +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/repository/implementation/FileDataRepositoryImpl.java @@ -412,7 +412,7 @@ private SBApplicationDTO getSbApplicationDTO(Map> nsWithApp String namespace = applicationDTO.getDeployPostfix(); String appName = applicationDTO.getVersion().split(":")[0]; String appVersion = applicationDTO.getVersion().replace(":", "-"); - String appFileRef = String.format("%s/%s", sharedData.getSbomsPath().get(), appVersion + ".sbom.json"); + String appFileRef = String.format("%s/%s/%s", sharedData.getSbomsPath().get(), appName, appVersion + ".sbom.json"); SBApplicationDTO dto = SBApplicationDTO.builder() .appName(appName) .appVersion(appVersion) diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/AdaptiveYaml.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/AdaptiveYaml.java new file mode 100644 index 000000000..61ca36c56 --- /dev/null +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/yaml/AdaptiveYaml.java @@ -0,0 +1,114 @@ +/* + * Copyright 2024-2025 NetCracker Technology Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.qubership.cloud.devops.cli.utils.yaml; + +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; + +public class AdaptiveYaml { + private static final int ALIAS_RATIO_RANGE_LOW = 400000; + private static final int ALIAS_RATIO_RANGE_HIGH = 4000000; + private static final double ALIAS_RATIO_RANGE = + (double) (ALIAS_RATIO_RANGE_HIGH - ALIAS_RATIO_RANGE_LOW); + + private static class Statistic { + int repeats = 0; + int complexity = 0; + } + + private static class Decoder { + // For more code details, check https://github.com/go-yaml/yaml/blob/v3/decode.go + + private final IdentityHashMap unique = new IdentityHashMap<>(); + private int decodeCount = 0; + private int aliasCount = 0; + + public int unmarshal(Object node) { + if (!(node instanceof Map || node instanceof List)) { + decodeCount++; + return 1; + } + + + if (unique.containsKey(node)) { + return alias(node); + } + + decodeCount++; + Statistic stat = new Statistic(); + stat.complexity = 1; + unique.put(node, stat); + + int childComplexity = 0; + + if (node instanceof Map map) { + for (Map.Entry e : map.entrySet()) { + childComplexity += unmarshal(e.getKey()); + childComplexity += unmarshal(e.getValue()); + } + } else if (node instanceof List list) { + for (Object item : list) { + childComplexity += unmarshal(item); + } + } + + stat.complexity += childComplexity; + + if (isLimitExceeded(aliasCount, decodeCount, node)) { + throw new RuntimeException("Excessive aliasing"); + } + + return stat.complexity; + } + + private int alias(Object node) { + Statistic stat = unique.get(node); + stat.repeats++; + decodeCount += stat.complexity; + aliasCount += stat.complexity; + return stat.complexity; + } + + private boolean isLimitExceeded(int alias, int decode, Object node ) { + double rt = allowedAliasRatio(decode); + double ad = ((double) alias / decode); + + return alias > 100 && + decode > 1000 && + ((double) alias / decode) > allowedAliasRatio(decode); + } + + private double allowedAliasRatio(int count) { + if (count <= ALIAS_RATIO_RANGE_LOW) return 0.99; + if (count >= ALIAS_RATIO_RANGE_HIGH) return 0.1; + + return 0.99 - 0.89 * ((double)(count - ALIAS_RATIO_RANGE_LOW) / ALIAS_RATIO_RANGE); + } + } + + public static boolean shouldExpand(Object data) { + Decoder decoder = new Decoder(); + try { + decoder.unmarshal(data); + } catch (RuntimeException e) { + return true; + } + return false; + } + +} diff --git a/build_effective_set_generator/effective-set-generator/src/test/java/org/qubership/cloud/devops/cli/CmdbCliTest.java b/build_effective_set_generator/effective-set-generator/src/test/java/org/qubership/cloud/devops/cli/CmdbCliTest.java index 46b3aba43..56fe74f12 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/java/org/qubership/cloud/devops/cli/CmdbCliTest.java +++ b/build_effective_set_generator/effective-set-generator/src/test/java/org/qubership/cloud/devops/cli/CmdbCliTest.java @@ -40,7 +40,8 @@ void testGenerateEffectiveSet(@TempDir Path tempDir) throws Exception { "--output", outputPath.toString(), "--effective-set-version", "v2.0", "--extra_params", "DEPLOYMENT_SESSION_ID=6d5a6ce9-0b55-429d-8877-f7a88dae3d9c", - "--app_chart_validation", "false" + "--app_chart_validation", "false", + "--custom-params", "@config.json" ); assertEquals(0, exitCode); diff --git a/build_effective_set_generator/effective-set-generator/src/test/java/org/qubership/cloud/devops/cli/utils/yaml/AdaptiveYamlTest.java b/build_effective_set_generator/effective-set-generator/src/test/java/org/qubership/cloud/devops/cli/utils/yaml/AdaptiveYamlTest.java new file mode 100644 index 000000000..69b19acb0 --- /dev/null +++ b/build_effective_set_generator/effective-set-generator/src/test/java/org/qubership/cloud/devops/cli/utils/yaml/AdaptiveYamlTest.java @@ -0,0 +1,64 @@ +/* + * Copyright 2024-2025 NetCracker Technology Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.qubership.cloud.devops.cli.utils.yaml; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class AdaptiveYamlTest { + @Test + void testNoAliasing() { + Map root = new HashMap<>(); + root.put("a", Map.of("x", 1)); + root.put("b", Map.of("y", 2)); + + assertFalse(AdaptiveYaml.shouldExpand(root)); + } + + @Test + void testSimpleAlias() { + Map shared = new HashMap<>(); + shared.put("x", 1); + + Map root = new HashMap<>(); + root.put("a", shared); + root.put("b", shared); + + assertFalse(AdaptiveYaml.shouldExpand(root)); + } + + @Test + void testExcessiveAliasing() { + Map shared = new HashMap<>(); + shared.put("x", 1); + + List list = new ArrayList<>(); + + for (int i = 0; i < 2000; i++) { + list.add(shared); + } + + assertTrue(AdaptiveYaml.shouldExpand(list)); + } +} diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/config.json b/build_effective_set_generator/effective-set-generator/src/test/resources/config.json new file mode 100644 index 000000000..1598906d6 --- /dev/null +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/config.json @@ -0,0 +1,8 @@ +{ + "deployment": { + "username": "${creds.get(\"minio-cred\").username}" + }, + "runtime": { + "password": "${STORAGE_PASSWORD}" + } +} diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring-origin/namespace.yml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring-origin/namespace.yml index d794157dc..b2f546b47 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring-origin/namespace.yml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring-origin/namespace.yml @@ -13,6 +13,14 @@ profile: name: "dev_bss_override" baseline: "dev" deployParameters: + server_port: 8080 + app_version: "3.0" + ssl_enabled: true + debug_mode_test: "true" + api_port: ${server_port} + service_version: ${app_version} + use_ssl: ${ssl_enabled} + log_level: ${debug_mode_test} ENVGENE_CONFIG_REF_NAME: "branch_name" ENVGENE_CONFIG_TAG: "No Ref tag" KMS_CERT_IN_BASE64: "${creds.get( \"kms-cert\" ).secret}" # paramset: test-deploy-creds version: 1 source: template @@ -36,6 +44,7 @@ deployParameters: TEST_ENVGENE_CREDS_GET_VAULT_SECRET_ID: "${creds.get( \"envgen-creds-get-vault-cred-secret-id\").secretId}" # paramset: test-deploy-creds version: 1 source: template TEST_SHARED_CREDS: "${creds.get('integration-cred').username}" # paramset: test-deploy-creds version: 1 source: template TEST_SHARED_CREDS_ACTIVATOR: "${creds.get('service-integration-cred').password}" # paramset: test-deploy-creds version: 1 source: template + TEST_CMDB_CREDS_USERNAME: "${cmdb.creds.get('creds-get-username-cred').username}" #added manually for cmdb.creds macro validation bss-app-exist: false core: # paramset: paramset-B version: 23.4 source: instance apps: diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/cloud.yml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/cloud.yml index 0f978c1c8..a8166f399 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/cloud.yml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/cloud.yml @@ -40,7 +40,7 @@ deployParameters: CDN_STORAGE_USERNAME: "${STORAGE_USERNAME}" # cloud passport: cluster-01 version: 1.5 CLOUD_DASHBOARD_URL: "https://dashboard.cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 CMDB_URL: "https://cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 - CONSUL_ENABLED: "true" # cloud passport: cluster-01 version: 1.5 + CONSUL_ENABLED: true # cloud passport: cluster-01 version: 1.5 DEFAULT_TENANT_ADMIN_LOGIN: "admin" # cloud passport: cluster-01 version: 1.5 DEFAULT_TENANT_ADMIN_PASSWORD: "password" # cloud passport: cluster-01 version: 1.5 DEFAULT_TENANT_NAME: "tenant" # cloud passport: cluster-01 version: 1.5 @@ -72,8 +72,10 @@ deployParameters: ZOOKEEPER_URL: "${ZOOKEEPER_ADDRESS}" # cloud passport: cluster-01 version: 1.5 e2eParameters: CLOUD_LEVEL_PARAM_1: "cloud-level-value-1" # paramset: cloud-level-params version: 25.1 source: instance -technicalConfigurationParameters: {} +technicalConfigurationParameters: + integrations.ndo-api-gw.url: "http://api-test:8080" + artifact: "org.snakeyaml" deployParameterSets: [] e2eParameterSets: [] technicalConfigurationParameterSets: [] -productionMode: false # cloud passport: cluster-01 version: 1.5 \ No newline at end of file +productionMode: false # cloud passport: cluster-01 version: 1.5 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/credentials.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/credentials.yaml index 7b4799025..0dfe41d91 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/credentials.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/credentials.yaml @@ -21,7 +21,9 @@ TEST_ENVGENE_CREDS_GET_VAULT_ROLE: envgeneNullValue TEST_ENVGENE_CREDS_GET_VAULT_SECRET_ID: envgeneNullValue TEST_SHARED_CREDS: user-placeholder-123 TEST_SHARED_CREDS_ACTIVATOR: pass-placeholder-123 +TEST_CMDB_CREDS_USERNAME: user-placeholder-123 graphite-remote-adapter: user-placeholder-123 kafka: password: pass-placeholder-123 username: user-placeholder-123 +password: pass-placeholder-123 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/parameters.yaml index 466dd98f9..d7f81b496 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/parameters.yaml @@ -12,7 +12,7 @@ CLOUD_PROTOCOL: https CLOUD_PUBLIC_HOST: cluster-01.qubership.org CMDB_URL: https://cluster-01.qubership.org CONSUL_ADMIN_TOKEN: token-placeholder-123 -CONSUL_ENABLED: 'true' +CONSUL_ENABLED: true CONSUL_PUBLIC_URL: http://consul.consul:8080 CONSUL_URL: http://consul.consul:8080 CONTROLLER_NAMESPACE: env-1-bg-controller @@ -73,9 +73,37 @@ TRACING_HOST: tracing-agent TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 +api_config: + connection: + host: db.example.com + port: 5432 +api_port: 8080 +app_version: '3.0' bss-app-exist: false core: apps: volumes: outputs: capacity: 20Gi +database_config: + connection: + host: db.example.com + port: 5432 +debug_mode_test: 'true' +log_level: 'true' +rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +server_port: 8080 +service_version: '3.0' +ssl_enabled: true +use_ssl: true +yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/pg/credentials.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/pg/credentials.yaml index f2b1a5208..0adb8aac0 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/pg/credentials.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/pg/credentials.yaml @@ -5,3 +5,4 @@ DOC_STORAGE_USERNAME: user-placeholder-123 K8S_TOKEN: token-placeholder-123 STORAGE_PASSWORD: pass-placeholder-123 STORAGE_USERNAME: user-placeholder-123 +password: pass-placeholder-123 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/pg/parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/pg/parameters.yaml index 38eb6ae59..6499c0b7d 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/pg/parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/pg/parameters.yaml @@ -12,7 +12,7 @@ CLOUD_PROTOCOL: https CLOUD_PUBLIC_HOST: cluster-01.qubership.org CMDB_URL: https://cluster-01.qubership.org CONSUL_ADMIN_TOKEN: token-placeholder-123 -CONSUL_ENABLED: 'true' +CONSUL_ENABLED: true CONSUL_PUBLIC_URL: http://consul.consul:8080 CONSUL_URL: http://consul.consul:8080 CUSTOM_HOST: cluster-01.qubership.org @@ -68,3 +68,23 @@ TRACING_HOST: tracing-agent TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 +api_config: + connection: + host: db.example.com + port: 5432 +database_config: + connection: + host: db.example.com + port: 5432 +rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/credentials.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/credentials.yaml index c45480d21..2f494be22 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/credentials.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/credentials.yaml @@ -28,6 +28,7 @@ TEST_ENVGENE_CREDS_GET_VAULT_ROLE: envgeneNullValue TEST_ENVGENE_CREDS_GET_VAULT_SECRET_ID: envgeneNullValue TEST_SHARED_CREDS: user-placeholder-123 TEST_SHARED_CREDS_ACTIVATOR: pass-placeholder-123 +TEST_CMDB_CREDS_USERNAME: user-placeholder-123 kafka: &id001 password: pass-placeholder-123 username: user-placeholder-123 @@ -62,6 +63,7 @@ global: &id002 TEST_ENVGENE_CREDS_GET_VAULT_SECRET_ID: envgeneNullValue TEST_SHARED_CREDS: user-placeholder-123 TEST_SHARED_CREDS_ACTIVATOR: pass-placeholder-123 + TEST_CMDB_CREDS_USERNAME: user-placeholder-123 graphite-remote-adapter: user-placeholder-123 kafka: *id001 alertmanager: *id002 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/custom-params.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/custom-params.yaml new file mode 100644 index 000000000..f2f7061e8 --- /dev/null +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/custom-params.yaml @@ -0,0 +1 @@ +username: user-placeholder-123 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/deployment-parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/deployment-parameters.yaml index d0aaf505b..32aa31035 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/deployment-parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/deployment-parameters.yaml @@ -17,7 +17,7 @@ CLOUD_PRIVATE_HOST: cluster-01.qubership.org CLOUD_PROTOCOL: https CLOUD_PUBLIC_HOST: cluster-01.qubership.org CMDB_URL: https://cluster-01.qubership.org -CONSUL_ENABLED: 'true' +CONSUL_ENABLED: true CONSUL_PUBLIC_URL: http://consul.consul:8080 CONSUL_URL: http://consul.consul:8080 CONTROLLER_NAMESPACE: env-1-bg-controller @@ -74,13 +74,41 @@ TRACING_HOST: tracing-agent TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 +api_config: &id001 + connection: + host: db.example.com + port: 5432 +api_port: 8080 +app_version: '3.0' bss-app-exist: false -core: &id001 +core: &id002 apps: volumes: outputs: capacity: 20Gi -global: &id002 +database_config: &id003 + connection: + host: db.example.com + port: 5432 +debug_mode_test: 'true' +log_level: 'true' +rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +server_port: 8080 +service_version: '3.0' +ssl_enabled: true +use_ssl: true +yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +global: &id004 API_DBAAS_ADDRESS: http://dbaas.dbaas:8080 APPLICATION_NAME: MONITORING ARTIFACT_DESCRIPTOR_ARTIFACT_ID: prod.platform.system.monitoring_monitoring-operator @@ -100,7 +128,7 @@ global: &id002 CLOUD_PROTOCOL: https CLOUD_PUBLIC_HOST: cluster-01.qubership.org CMDB_URL: https://cluster-01.qubership.org - CONSUL_ENABLED: 'true' + CONSUL_ENABLED: true CONSUL_PUBLIC_URL: http://consul.consul:8080 CONSUL_URL: http://consul.consul:8080 CONTROLLER_NAMESPACE: env-1-bg-controller @@ -157,50 +185,72 @@ global: &id002 TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 + api_config: *id001 + api_port: 8080 + app_version: '3.0' bss-app-exist: false - core: *id001 -alertmanager: *id002 -blackbox-exporter: *id002 -cert-exporter: *id002 -cloud-events-exporter: *id002 -cloud-events-reader: *id002 -cloudwatch-exporter: *id002 -common-dashboards: *id002 -configmap-reload: *id002 -configurations-streamer: *id002 -grafana: *id002 -grafana-image-renderer: *id002 -grafana-operator: *id002 -grafana-plugins-init: *id002 -graphite-remote-adapter: *id002 -json-exporter: *id002 -kube-rbac-proxy: *id002 -kube-state-metrics: *id002 -monitoring-operator: *id002 -network-latency-exporter: *id002 -node-exporter: *id002 -oauth2-proxy: *id002 -platform_monitoring_tests: *id002 -prometheus: *id002 -prometheus-adapter: *id002 -prometheus-adapter-converter: *id002 -prometheus-adapter-operator: *id002 -prometheus-config-reloader: *id002 -prometheus-operator: *id002 -promitor-agent-resource-discovery: *id002 -promitor-agent-scraper: *id002 -promxy: *id002 -pushgateway: *id002 -stackdriver-exporter: *id002 -version-exporter: *id002 -victoriametrics-operator: *id002 -vmagent: *id002 -vmalert: *id002 -vmauth: *id002 -vmcleanup: *id002 -vminsert: *id002 -vmoperator: *id002 -vmoperator_config_reloader: *id002 -vmselect: *id002 -vmsingle: *id002 -vmstorage: *id002 + core: *id002 + database_config: *id003 + debug_mode_test: 'true' + log_level: 'true' + rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 + server_port: 8080 + service_version: '3.0' + ssl_enabled: true + use_ssl: true + yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +alertmanager: *id004 +blackbox-exporter: *id004 +cert-exporter: *id004 +cloud-events-exporter: *id004 +cloud-events-reader: *id004 +cloudwatch-exporter: *id004 +common-dashboards: *id004 +configmap-reload: *id004 +configurations-streamer: *id004 +grafana: *id004 +grafana-image-renderer: *id004 +grafana-operator: *id004 +grafana-plugins-init: *id004 +graphite-remote-adapter: *id004 +json-exporter: *id004 +kube-rbac-proxy: *id004 +kube-state-metrics: *id004 +monitoring-operator: *id004 +network-latency-exporter: *id004 +node-exporter: *id004 +oauth2-proxy: *id004 +platform_monitoring_tests: *id004 +prometheus: *id004 +prometheus-adapter: *id004 +prometheus-adapter-converter: *id004 +prometheus-adapter-operator: *id004 +prometheus-config-reloader: *id004 +prometheus-operator: *id004 +promitor-agent-resource-discovery: *id004 +promitor-agent-scraper: *id004 +promxy: *id004 +pushgateway: *id004 +stackdriver-exporter: *id004 +version-exporter: *id004 +victoriametrics-operator: *id004 +vmagent: *id004 +vmalert: *id004 +vmauth: *id004 +vmcleanup: *id004 +vminsert: *id004 +vmoperator: *id004 +vmoperator_config_reloader: *id004 +vmselect: *id004 +vmsingle: *id004 +vmstorage: *id004 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/custom-params.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/custom-params.yaml new file mode 100644 index 000000000..f2f7061e8 --- /dev/null +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/custom-params.yaml @@ -0,0 +1 @@ +username: user-placeholder-123 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deployment-parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deployment-parameters.yaml index c11ace9fc..a58486edd 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deployment-parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deployment-parameters.yaml @@ -17,7 +17,7 @@ CLOUD_PRIVATE_HOST: cluster-01.qubership.org CLOUD_PROTOCOL: https CLOUD_PUBLIC_HOST: cluster-01.qubership.org CMDB_URL: https://cluster-01.qubership.org -CONSUL_ENABLED: 'true' +CONSUL_ENABLED: true CONSUL_PUBLIC_URL: http://consul.consul:8080 CONSUL_URL: http://consul.consul:8080 CUSTOM_HOST: cluster-01.qubership.org @@ -69,7 +69,27 @@ TRACING_HOST: tracing-agent TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 -global: &id001 +api_config: &id001 + connection: + host: db.example.com + port: 5432 +database_config: &id002 + connection: + host: db.example.com + port: 5432 +rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +global: &id003 API_DBAAS_ADDRESS: http://dbaas.dbaas:8080 APPLICATION_NAME: postgres ARTIFACT_DESCRIPTOR_ARTIFACT_ID: prod.platform.ha.postgres @@ -89,7 +109,7 @@ global: &id001 CLOUD_PROTOCOL: https CLOUD_PUBLIC_HOST: cluster-01.qubership.org CMDB_URL: https://cluster-01.qubership.org - CONSUL_ENABLED: 'true' + CONSUL_ENABLED: true CONSUL_PUBLIC_URL: http://consul.consul:8080 CONSUL_URL: http://consul.consul:8080 CUSTOM_HOST: cluster-01.qubership.org @@ -141,10 +161,24 @@ global: &id001 TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 -patroni-core: *id001 -pg_patroni: *id001 -pg_upgrade: *id001 -pgbackrest_sidecar: *id001 -postgres_operator_init: *id001 -postgres_operator_tests: *id001 -vault_env: *id001 + api_config: *id001 + database_config: *id002 + rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 + yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +patroni-core: *id003 +pg_patroni: *id003 +pg_upgrade: *id003 +pgbackrest_sidecar: *id003 +postgres_operator_init: *id003 +postgres_operator_tests: *id003 +vault_env: *id003 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring-origin/MONITORING/credentials.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring-origin/MONITORING/credentials.yaml index e69de29bb..0def499f3 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring-origin/MONITORING/credentials.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring-origin/MONITORING/credentials.yaml @@ -0,0 +1 @@ +password: pass-placeholder-123 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring-origin/MONITORING/parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring-origin/MONITORING/parameters.yaml index 790b85de6..0e3a1acf8 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring-origin/MONITORING/parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring-origin/MONITORING/parameters.yaml @@ -1,3 +1,5 @@ PARAM_2: value-2 PARAM_6: value-6 TECHNICAL_PARAM_1: VALUE_TP_1 +integrations.ndo-api-gw.url: "http://api-test:8080" +artifact: "org.snakeyaml" diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/pg/postgres/credentials.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/pg/postgres/credentials.yaml index e69de29bb..0def499f3 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/pg/postgres/credentials.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/pg/postgres/credentials.yaml @@ -0,0 +1 @@ +password: pass-placeholder-123 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/pg/postgres/parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/pg/postgres/parameters.yaml index e69de29bb..87987d319 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/pg/postgres/parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/pg/postgres/parameters.yaml @@ -0,0 +1,2 @@ +integrations.ndo-api-gw.url: "http://api-test:8080" +artifact: "org.snakeyaml" diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/credentials.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/credentials.yaml index 38c4c40a6..511db0adf 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/credentials.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/credentials.yaml @@ -3,7 +3,5 @@ bg_domain: password: pass-placeholder-123 username: user-placeholder-123 k8s_tokens: - monitoring-origin: token-placeholder-123 - pg: token-placeholder-123 pl-01-pg: token-placeholder-123 pl-01-monitoring: token-placeholder-123 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/tenant.yml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/tenant.yml index c526990a0..b35359480 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/tenant.yml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/tenant.yml @@ -15,4 +15,15 @@ globalE2EParameters: mergeTenantsAndE2EParameters: false environmentParameters: {} deployParameters: - ESCAPE_SEQUENCE: "true" \ No newline at end of file + api_config: ${database_config} + database_config: + connection: + host: db.example.com + port: 5432 + yaml_template: | + services: + api: + image: api:latest + ports: + - 8080:8080 + rendered_template: ${yaml_template} diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/sboms/MONITORING-0.91.0-delivery-20241115.141230-4-RELEASE.sbom.json b/build_effective_set_generator/effective-set-generator/src/test/resources/sboms/MONITORING/MONITORING-0.91.0-delivery-20241115.141230-4-RELEASE.sbom.json similarity index 100% rename from build_effective_set_generator/effective-set-generator/src/test/resources/sboms/MONITORING-0.91.0-delivery-20241115.141230-4-RELEASE.sbom.json rename to build_effective_set_generator/effective-set-generator/src/test/resources/sboms/MONITORING/MONITORING-0.91.0-delivery-20241115.141230-4-RELEASE.sbom.json diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/sboms/postgres-pg16-2.10.5-20241215.141230-3-RELEASE.sbom.json b/build_effective_set_generator/effective-set-generator/src/test/resources/sboms/postgres/postgres-pg16-2.10.5-20241215.141230-3-RELEASE.sbom.json similarity index 100% rename from build_effective_set_generator/effective-set-generator/src/test/resources/sboms/postgres-pg16-2.10.5-20241215.141230-3-RELEASE.sbom.json rename to build_effective_set_generator/effective-set-generator/src/test/resources/sboms/postgres/postgres-pg16-2.10.5-20241215.141230-3-RELEASE.sbom.json diff --git a/build_effective_set_generator/parameter-calculator-bom/pom.xml b/build_effective_set_generator/parameter-calculator-bom/pom.xml index 3c095505d..1e4f6b5bf 100644 --- a/build_effective_set_generator/parameter-calculator-bom/pom.xml +++ b/build_effective_set_generator/parameter-calculator-bom/pom.xml @@ -55,7 +55,7 @@ org.yaml snakeyaml - 2.2 + 2.3 org.eclipse.jgit diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/ParametersProcessor.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/ParametersProcessor.java index 1777771f2..34725582a 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/ParametersProcessor.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/ParametersProcessor.java @@ -22,7 +22,6 @@ import org.qubership.cloud.parameters.processor.dto.Params; import org.qubership.cloud.parameters.processor.expression.ExpressionLanguage; import org.qubership.cloud.parameters.processor.expression.Language; -import org.qubership.cloud.parameters.processor.expression.PlainLanguage; import org.qubership.cloud.parameters.processor.expression.binding.Binding; import jakarta.inject.Inject; import jakarta.inject.Singleton; @@ -43,16 +42,12 @@ public ParametersProcessor(OpenTelemetryProvider openTelemetryProvider) { this.openTelemetryProvider = openTelemetryProvider; } - public Params processAllParameters(String tenant, String cloud, String namespace, String application, String defaultEscapeSequence, DeployerInputs deployerInputs, String originalNamespace) { + public Params processAllParameters(String tenant, String cloud, String namespace, String application, + DeployerInputs deployerInputs, String originalNamespace, Map customParams) { return openTelemetryProvider.withSpan("process", () -> { - Binding binding = new Binding(defaultEscapeSequence, deployerInputs).init(tenant, cloud, namespace, application, originalNamespace); + Binding binding = new Binding(deployerInputs).init(tenant, cloud, namespace, application, originalNamespace, customParams); Language lang; - if (binding.getProcessorType().equals("true")) { - lang = new ExpressionLanguage(binding); - } else { - lang = new PlainLanguage(binding); - } - + lang = new ExpressionLanguage(binding); Map deploy = lang.processDeployment(); Map tech = lang.processConfigServerApp(); binding.additionalParameters(deploy); @@ -60,31 +55,21 @@ public Params processAllParameters(String tenant, String cloud, String namespace }); } - public Params processE2EParameters(String tenant, String cloud, String namespace, String application, String defaultEscapeSequence, DeployerInputs deployerInputs, String originalNamespace) { + public Params processE2EParameters(String tenant, String cloud, String namespace, String application, DeployerInputs deployerInputs, String originalNamespace) { return openTelemetryProvider.withSpan("process", () -> { - Binding binding = new Binding(defaultEscapeSequence, deployerInputs).init(tenant, cloud, namespace, application, originalNamespace); + Binding binding = new Binding(deployerInputs).init(tenant, cloud, namespace, application, originalNamespace, new HashMap<>()); Language lang; - if (binding.getProcessorType().equals("true")) { - lang = new ExpressionLanguage(binding); - } else { - lang = new PlainLanguage(binding); - } - + lang = new ExpressionLanguage(binding); Map e2e = lang.processCloudE2E(); return Params.builder().e2eParams(e2e).build(); }); } - public Params processNamespaceParameters(String tenant, String cloud, String namespace, String defaultEscapeSequence, DeployerInputs deployerInputs, String originalNamespace) { + public Params processNamespaceParameters(String tenant, String cloud, String namespace, DeployerInputs deployerInputs, String originalNamespace) { return openTelemetryProvider.withSpan("process", () -> { - Binding binding = new Binding(defaultEscapeSequence, deployerInputs).init(tenant, cloud, namespace, null, originalNamespace); + Binding binding = new Binding(deployerInputs).init(tenant, cloud, namespace, null, originalNamespace, new HashMap<>()); Language lang; - if (binding.getProcessorType().equals("true")) { - lang = new ExpressionLanguage(binding); - } else { - lang = new PlainLanguage(binding); - } - + lang = new ExpressionLanguage(binding); Map namespaceParams = lang.processNamespace(); binding.additionalParameters(namespaceParams); return Params.builder().cleanupParams(namespaceParams).build(); @@ -93,14 +78,10 @@ public Params processNamespaceParameters(String tenant, String cloud, String nam public Map processParameters(Map parameters) { return openTelemetryProvider.withSpan("process", () -> { - Binding binding = new Binding("true"); + Binding binding = new Binding(); binding.put("creds", new Parameter(new CredentialsMap(binding).init())); Language lang; - if (binding.getProcessorType().equals("true")) { - lang = new ExpressionLanguage(binding); - } else { - lang = new PlainLanguage(binding); - } + lang = new ExpressionLanguage(binding); return lang.processParameters(parameters); }); diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/dto/ParameterBundle.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/dto/ParameterBundle.java index 86cd3efd5..0041ea994 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/dto/ParameterBundle.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/dto/ParameterBundle.java @@ -36,6 +36,8 @@ public class ParameterBundle { Map collisionSecureParameters; Map cleanupParameters; Map cleanupSecureParameters; + Map customDeployParameters; + Map customTechParameters; boolean processPerServiceParams; String appChartName; } diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/ExpressionLanguage.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/ExpressionLanguage.java index cb7a571c0..328b50068 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/ExpressionLanguage.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/ExpressionLanguage.java @@ -16,13 +16,13 @@ package org.qubership.cloud.parameters.processor.expression; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; import com.hubspot.jinjava.Jinjava; import com.hubspot.jinjava.JinjavaConfig; +import com.hubspot.jinjava.interpret.Context; import com.hubspot.jinjava.interpret.FatalTemplateErrorsException; +import com.hubspot.jinjava.interpret.JinjavaInterpreter; import groovy.lang.GroovyClassLoader; import groovy.lang.GroovyRuntimeException; import groovy.text.GStringTemplateEngine; @@ -35,7 +35,6 @@ import org.qubership.cloud.devops.gstringtojinjavatranslator.jinjava.*; import org.qubership.cloud.devops.gstringtojinjavatranslator.translator.GStringToJinJavaTranslator; import org.qubership.cloud.parameters.processor.MergeMap; -import org.qubership.cloud.parameters.processor.ParametersProcessor; import org.qubership.cloud.parameters.processor.exceptions.ExpressionLanguageException; import org.qubership.cloud.parameters.processor.expression.binding.Binding; import org.qubership.cloud.parameters.processor.expression.binding.DynamicMap; @@ -58,11 +57,17 @@ public class ExpressionLanguage extends AbstractLanguage { private static final Pattern EXPRESSION_PATTERN = Pattern.compile("(?$") // <% VARIABLE %> + }; private boolean insecure; private final Jinjava jinjava; private final GStringToJinJavaTranslator gStringToJinJavaTranslator; + private final DynamicPropertyResolver dynamicResolver; public ExpressionLanguage(Binding binding) { super(binding); @@ -80,6 +85,8 @@ public ExpressionLanguage(Binding binding) { this.binding.forEach((key1, value) -> this.binding.put(key1, translateParameter(value.getValue()))); + + this.dynamicResolver = new DynamicPropertyResolver(this.binding); } private Parameter translateParameter(Object value) { @@ -211,29 +218,32 @@ private Parameter processValue(Object value, Map binding, boo parameter.setValue(removeEscaping(escapeDollar, parameter.getValue())); return parameter; } + Object val = getValue(value); boolean isProcessed = false; boolean isSecured = getIsSecured(value); if (val instanceof String) { - String strValue = (String) val; + Parameter preserved = tryPreserveType(value, binding); + if (preserved != null) { + return preserved; + } + String strValue = (String) val; - String rendered = ""; - this.binding.getTypeCollector().clear(); + String jinJavaRendered = ""; try { - rendered = renderStringByJinJava(strValue, binding, escapeDollar); + jinJavaRendered = renderStringByJinJava(strValue, binding, escapeDollar); + val = jinJavaRendered; } catch (Exception e) { - logDebug(String.format("Parameter {} was not processed by JinJava, hence reverting to Groovy.", strValue)); - rendered = renderStringByGroovy(strValue, binding, escapeDollar); + log.debug(String.format("Parameter {} was not processed by JinJava, hence reverting to Groovy.", strValue)); + String groovyRendered = renderStringByGroovy(strValue, binding, escapeDollar); + val = groovyRendered; } - Object originalValue = this.binding.getTypeCollector().get(rendered); // Object - Class targetType = (originalValue != null) ? (Class) originalValue : String.class; - val = convertToType(rendered, targetType); isProcessed = true; - Matcher secureMarkerMatcher = SECURED_PATTERN.matcher(String.valueOf(val)); + Matcher secureMarkerMatcher = SECURED_PATTERN.matcher((String) val); if (secureMarkerMatcher.find()) { isSecured = true; val = ((String) Objects.requireNonNull(val)).replaceAll("([\\u0096\\u0097])", ""); @@ -250,6 +260,88 @@ private Parameter processValue(Object value, Map binding, boo return ret; } + // Extracts variable name from simple parameter reference patterns: ${VAR}, $VAR, or <% VAR %>. + private String extractParameterReference(String value) { + if (value == null) { + return null; + } + String trimmed = value.trim(); + for (Pattern pattern : PARAMETER_REFERENCE_PATTERNS) { + Matcher matcher = pattern.matcher(trimmed); + if (matcher.matches()) { + return matcher.group(1); + } + } + return null; + } + + // Preserves data type for simple parameter references (${VAR}, $VAR) using Jinjava's expression resolver. + // Returns null if type preservation is not applicable (complex expressions or String values). + private Parameter tryPreserveType(Object value, Map binding) { + Object val = getValue(value); + if (!(val instanceof String)) { + return null; + } + + String referencedVar = extractParameterReference((String) val); + if (referencedVar == null) { + return null; + } + + // Use Jinjava's expression resolver - returns typed Object (Integer, Boolean, etc.) + Object resolvedValue = resolveWithJinjava(referencedVar, binding); + + // Unwrap if still a Parameter + if (resolvedValue instanceof Parameter) { + resolvedValue = ((Parameter) resolvedValue).getValue(); + } + + if (resolvedValue == null || resolvedValue instanceof String) { + return null; + } + + // Get secured flag from source parameter + Parameter srcParam = binding.get(referencedVar); + if (srcParam == null) { + srcParam = this.binding.get(referencedVar); + } + boolean isSecured = srcParam != null && srcParam.isSecured(); + + Parameter result = new Parameter(resolvedValue); + if (value instanceof Parameter) { + result.setOrigin(((Parameter) value).getOrigin()); + } + result.setParsed(true); + result.setProcessed(true); + result.setSecured(isSecured); + result.setValid(true); + + log.debug("Type preserved for {}: {} ({})", referencedVar, resolvedValue, resolvedValue.getClass().getSimpleName()); + return result; + } + + // Uses Jinjava's expression resolver which returns typed Object instead of String. + private Object resolveWithJinjava(String expression, Map binding) { + try { + Context context = new Context(jinjava.getGlobalContextCopy(), binding, jinjava.getGlobalConfig().getDisabled()); + context.setDynamicVariableResolver(this.dynamicResolver); + + JinjavaInterpreter interpreter = jinjava.getGlobalConfig() + .getInterpreterFactory() + .newInstance(jinjava, context, jinjava.getGlobalConfig()); + + JinjavaInterpreter.pushCurrent(interpreter); + try { + return interpreter.resolveELExpression(expression, -1); + } finally { + JinjavaInterpreter.popCurrent(); + } + } catch (Exception e) { + log.debug("Failed to resolve '{}' with Jinjava: {}", expression, e.getMessage()); + return null; + } + } + private String renderStringByGroovy(String value, Map binding, boolean escapeDollar) { int i = 0; String rendered = value; @@ -280,21 +372,14 @@ private String renderStringByJinJava(String value, Map bindin return rendered; } - private Object removeEscaping(boolean escapeDollar, Object val) throws JsonProcessingException { - - if (escapeDollar && val != null) { - Class originalType = val.getClass(); - String strValue; - if (val instanceof String) { - strValue = val.toString(); - } else { - strValue = mapper.writeValueAsString(val); - } - strValue = strValue.replaceAll("\\\\\\$", "\\$"); // \$ -> $ - strValue = strValue.replaceAll("\\\\\\\\", "\\\\"); // \\ -> \ - return convertToType(strValue, originalType); + private Object removeEscaping(boolean escapeDollar, Object val) { + // Only process escaping for String values - non-String types (Integer, Boolean, etc.) + // don't need escape processing and should preserve their original type + if (escapeDollar && val instanceof String) { + String strValue = (String) val; + strValue = strValue.replaceAll("\\\\\\$", "\\$"); // \$ -> $ + val = strValue.replaceAll("\\\\\\\\", "\\\\"); // \\ -> \ } - return val; } @@ -344,9 +429,6 @@ private Map processMap(Map map, Map processParameters(Map parameters) return processedParams; } - private static Object convertToType(String value, Class type) { - if (type == String.class) { - return value; - } else if (type == Integer.class || type == int.class) { - return Integer.parseInt(value); - } else if (type == Long.class || type == long.class) { - return Long.parseLong(value); - } else if (type == Boolean.class || type == boolean.class) { - return Boolean.parseBoolean(value); - } else if (type == Double.class || type == double.class) { - return Double.parseDouble(value); - } - return value; - } } diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/PlainLanguage.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/PlainLanguage.java deleted file mode 100644 index 33067aba8..000000000 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/PlainLanguage.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright 2024-2025 NetCracker Technology Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.qubership.cloud.parameters.processor.expression; - -import org.qubership.cloud.parameters.processor.MergeMap; -import org.qubership.cloud.parameters.processor.expression.binding.Binding; -import org.qubership.cloud.parameters.processor.expression.binding.DynamicMap; -import org.qubership.cloud.parameters.processor.expression.binding.EscapeMap; -import org.qubership.cloud.devops.commons.utils.Parameter; - -import java.util.AbstractMap; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class PlainLanguage extends AbstractLanguage { - - public PlainLanguage(Binding binding) { - super(binding); - } - - @Override - protected Map createMap() { - return new HashMap<>(); - } - - private Object getValue(Object object) { - if (object instanceof Parameter) { - return ((Parameter) object).getValue(); - } else { - return object; - } - } - - private Parameter processValue(Object value) { - Object val = getValue(value); - if (val instanceof List) { - val = processList((List) val); - } else if (val instanceof Map) { - val = processMap((Map) val); - } - Parameter ret = new Parameter(value); - ret.setValue(val); - return ret; - } - - protected Map processMap(Map map) { - if (map == null) { - map = Collections.emptyMap(); - } - return map.entrySet().stream() - .filter(x -> getValue(x.getValue()) instanceof String || !(getValue(x.getValue()) instanceof DynamicMap || getValue(x.getValue()) instanceof EscapeMap)) - .collect(Collectors.toMap(Map.Entry::getKey, e -> processValue(e.getValue()))); - } - - private List processList(List list) { - return list.stream().map(this::processValue).collect(Collectors.toList()); - } - - protected Map processMap(String mapName) { - return processMap(binding.setDefault(mapName)); - } - - private Object convertParameterToObject(Object value) { - if (value instanceof Parameter) { - value = ((Parameter) value).getValue(); - } - if (value instanceof Map) { - value = convertParameterMapToObject((Map) value); - } else if (value instanceof List) { - value = convertParameterListToObject((List) value); - } - return value; - } - - private List convertParameterListToObject(List list) { - return list.stream() - .map(this::convertParameterToObject) - .collect(Collectors.toList()); - } - - private Map convertParameterMapToObject(Map map) { - return map.entrySet().stream() - .map(entry -> new AbstractMap.SimpleEntry<>(entry.getKey(), entry.getValue().getValue())) - .collect(HashMap::new, (m, v) -> m.put(v.getKey(), convertParameterToObject(v.getValue())), HashMap::putAll); - } - - public Map processDeployment() { - Map result = new MergeMap(); - - Map override = new HashMap<>(); - processNamespaceApp(override); - - //merge only overall cloud and custom params - result.putAll(override); - result.putAll(processMap("")); - - return result; - } - - - @Override - public Map processE2E() { - Map result = new HashMap<>(); - processE2E(result); - - return result; - } - - @Override - public Map processCloudE2E() { - Map result = new HashMap<>(); - processCloudE2E(result); - - return result; - } - - @Override - public Map processNamespaceApp() { - Map result = new HashMap<>(); - - processNamespaceApp(result); - return processMap(result); - } - - @Override - public Map processNamespace() { - Map result = new MergeMap(); - - processNamespace(result); - - return processMap(result); - } - - @Override - public Map processConfigServerApp() { - return processNamespaceAppConfigServer(); - } - - @Override - public Map processNamespaceAppConfigServer() { - Map result = new HashMap<>(); - - processNamespaceAppConfigServer(result); - return processMap(result); - } - @Override - public Map processParameters(Map parameters) { - return new HashMap<>(); - } - -} diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/ApplicationMap.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/ApplicationMap.java index b86048bcc..dec252e9d 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/ApplicationMap.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/ApplicationMap.java @@ -39,7 +39,6 @@ public Map getMap(String key) { Application config = Injector.getInstance().getDi().get(ApplicationService.class).getByName(key, namespace); if (config != null) { Map map = new EscapeMap(config.getParams(), binding, String.format(ParametersConstants.APP_ORIGIN, key)); - checkEscape(map); maps.put(key, map); return map; } diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/Binding.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/Binding.java index c6c3bec29..e7a9160f6 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/Binding.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/Binding.java @@ -26,7 +26,6 @@ import org.qubership.cloud.devops.commons.utils.Parameter; import org.qubership.cloud.parameters.processor.dto.DeployerInputs; import org.qubership.cloud.parameters.processor.parser.EscapeParametersParser; -import org.qubership.cloud.parameters.processor.parser.OldParametersParser; import org.qubership.cloud.parameters.processor.parser.ParametersParser; import lombok.Getter; @@ -38,7 +37,6 @@ @Slf4j public class Binding extends HashMap implements Cloneable { - String escapeSequence; @Getter private DeployerInputs deployerInputs; private Map defaultMap; @@ -46,32 +44,20 @@ public class Binding extends HashMap implements Cloneable { @Getter private String tenant; private ParametersParser escapeParser; - private ParametersParser oldParser; - @Getter - private Map> typeCollector = new HashMap<>(); - public Binding(String defaultEscapeSequence) { - this.escapeSequence = defaultEscapeSequence; + public Binding() { this.tenant = ""; } - public Binding(String defaultEscapeSequence, DeployerInputs deployerInputs) { - this.escapeSequence = defaultEscapeSequence; + public Binding(DeployerInputs deployerInputs) { this.tenant = ""; this.deployerInputs = deployerInputs; } - public String getProcessorType() { - return escapeSequence; - } public ParametersParser getParser() { ParametersParser parser; - if (escapeSequence.equals("true")) { - return (parser = escapeParser) == null ? (escapeParser = new EscapeParametersParser()) : parser; - } else { - return (parser = oldParser) == null ? (oldParser = new OldParametersParser()) : parser; - } + return (parser = escapeParser) == null ? (escapeParser = new EscapeParametersParser()) : parser; } private void processSet(String tenant, String setName, String application, EscapeMap parameterSet, EscapeMap applicationMap) { @@ -88,15 +74,20 @@ private void processSet(String tenant, String setName, String application, Escap } } - public Binding init(String tenant, String cloud, String namespace, String application, String originalNamespace) { + public Binding init(String tenant, String cloud, String namespace, String application, + String originalNamespace, Map customParams) { this.tenant = tenant; super.put("tenant", new Parameter(new TenantMap(tenant, cloud, namespace, application, this, originalNamespace).init())); super.put("application", new Parameter(new ApplicationMap(application, this, namespace).init())); super.put("creds", new Parameter(new CredentialsMap(this).init())); + EscapeMap credentials = new EscapeMap(null, this, ""); + credentials.put("creds", new CredentialsMap(this).init()); + super.put("cmdb",new Parameter(credentials)); - Map processed = calculateCredentialsAndPrepareStructuredParams(this, Boolean.parseBoolean(getProcessorType())); + Map processed = calculateCredentialsAndPrepareStructuredParams(this); this.putAll(processed); + this.putAll(customParams); return this; } @@ -105,7 +96,7 @@ public Binding additionalParameters(Map parameters) { return this; } - private Map calculateCredentialsAndPrepareParams(String keyCloudParameter, Parameter valueCloudParameter, boolean escapeSequence) { + private Map calculateCredentialsAndPrepareParams(String keyCloudParameter, Parameter valueCloudParameter) { Object value = valueCloudParameter.getValue(); if (value instanceof String) { value = ((String) value).trim(); @@ -148,17 +139,17 @@ private Map calculateCredentialsAndPrepareParams(String keyCl } } - private Map calculateCredentialsAndPrepareStructuredParams(Map params, Boolean escape) { + private Map calculateCredentialsAndPrepareStructuredParams(Map params) { Map result = params.entrySet().stream().map(entry -> { Parameter param = new Parameter(entry.getValue()); if (param.getValue() instanceof Map) { return new HashMap() { { - put(entry.getKey(), new Parameter(calculateCredentialsAndPrepareStructuredParams((Map) param.getValue(), escape))); + put(entry.getKey(), new Parameter(calculateCredentialsAndPrepareStructuredParams((Map) param.getValue()))); } }; } - return calculateCredentialsAndPrepareParams(entry.getKey(), param, escape); + return calculateCredentialsAndPrepareParams(entry.getKey(), param); }).flatMap(c -> c.entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (e1, e2) -> e2)); params.clear(); @@ -212,9 +203,6 @@ public Parameter get(Object key) { if (result == null || result.getValue() == null) { return null; } - if (result != null && result.getValue() != null) { - typeCollector.put(result.getValue().toString(), result.getValue().getClass()); - } return result; } diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/CloudApplicationMap.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/CloudApplicationMap.java index 1d526b3c9..c870e8925 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/CloudApplicationMap.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/CloudApplicationMap.java @@ -67,10 +67,6 @@ public Map getMap(String appName) { }); } - checkEscape(map); - checkEscape(parameterSetMap); - checkEscape(configServerMap); - checkEscape(parameterSetConfigServerMap); map.put("config-server", configServerMap); maps.put(appName, map); return map; diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/CloudMap.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/CloudMap.java index 0fb3c2485..4334a6e1c 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/CloudMap.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/CloudMap.java @@ -80,10 +80,6 @@ public Map getMap(String cloudName) { map.put("app", new Parameter(new CloudApplicationMap(config, defaultApp, binding).init())); - - checkEscape(map); - checkEscape(e2e); - checkEscape(configServer); CredentialUtils credentialUtils = Injector.getInstance().getCredentialUtils(); for (DBaaS dbaas : config.getDbaasCfg()) { if (dbaas.getApiUrl() != null) { diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/DynamicMap.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/DynamicMap.java index 82f7342e5..72785a877 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/DynamicMap.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/DynamicMap.java @@ -40,12 +40,6 @@ public abstract class DynamicMap implements Map, Serializable public abstract Map getMap(String key); - protected void checkEscape(Map map) { - Object processor = map.get("ESCAPE_SEQUENCE").getValue(); - if (processor != null) { - binding.escapeSequence = processor.toString(); - } - } public DynamicMap init() { if (defaultMap != null) { diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/NamespaceApplicationMap.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/NamespaceApplicationMap.java index 8c164cb23..1d3683fff 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/NamespaceApplicationMap.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/NamespaceApplicationMap.java @@ -65,8 +65,6 @@ public Map getMap(String appName) { map.put("APPLICATION_NAME", appName); - checkEscape(map); - checkEscape(configServerMap); map.put("config-server", configServerMap); try { if (binding.getDeployerInputs() != null && binding.getDeployerInputs().getAppVersion() != null) { diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/NamespaceMap.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/NamespaceMap.java index 9d2ce0dca..74d9f1fe6 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/NamespaceMap.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/NamespaceMap.java @@ -204,10 +204,6 @@ public Map getMap(String namespaceName) { EscapeMap e2e = new EscapeMap(config.getE2eParameters(), binding, String.format(ParametersConstants.NS_E2E_ORIGIN, tenant, this.cloud, namespaceName)); EscapeMap configServer = new EscapeMap(config.getConfigServerParameters(), binding, String.format(ParametersConstants.NS_CONFIG_SERVER_ORIGIN, tenant, this.cloud, namespaceName)); - checkEscape(map); - checkEscape(e2e); - checkEscape(configServer); - map.put(E2E, new Parameter(e2e)); map.put(CONFIG_SERVER, new Parameter(configServer)); maps.put(namespaceName, map); diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/TenantMap.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/TenantMap.java index 3ac5dcda5..acde3c677 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/TenantMap.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/TenantMap.java @@ -64,10 +64,6 @@ public Map getMap(String tenantName) { EscapeMap e2e = new EscapeMap(config.getGlobalParameters().getE2eParameters().getEnvParameters(), binding, String.format(ParametersConstants.TENANT_E2E_ORIGIN, tenantName)); EscapeMap configServer = new EscapeMap(config.getGlobalParameters().getTechnicalConfiguration(), binding, String.format(ParametersConstants.TENANT_CONFIG_SERVER_ORIGIN, tenantName)); - checkEscape(map); - checkEscape(e2e); - checkEscape(configServer); - map.put("cloud", new Parameter(new CloudMap(tenantName, defaultCloud, defaultNamespace, defaultApp, binding, originalNamespace).init())); map.put("e2e", new Parameter(e2e)); map.put("TENANTNAME", tenantName); diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/parser/OldParametersParser.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/parser/OldParametersParser.java deleted file mode 100644 index 1bbe8b1e6..000000000 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/parser/OldParametersParser.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2024-2025 NetCracker Technology Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.qubership.cloud.parameters.processor.parser; - -import org.qubership.cloud.devops.commons.utils.constant.ParametersConstants; -import org.qubership.cloud.devops.commons.utils.Parameter; - -import groovy.json.JsonSlurperClassic; - -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; - -public class OldParametersParser implements ParametersParser, Serializable { - - @Override - public Object processParam(String param) { - // auto escape symbols \ and " for CRM guys - String valueOfParam = param.replaceAll("(\"|\\\\)", "\\\\$0").trim(); - if (valueOfParam.startsWith("\'{")) { - String jsString = valueOfParam.substring(1, valueOfParam.length() - 1); - return new JsonSlurperClassic().parseText(jsString.replaceAll("\\\\", "")); - } else { - return valueOfParam; - } - } - - @Override - public Map parse(String customParams) { - Map k = new HashMap<>(); - - for (String paramLine : customParams.split(";")) { - if (!paramLine.trim().isEmpty()) { - String[] pairs = paramLine.split("=", 2); - - if (!paramLine.contains("=") || pairs[0].isEmpty()) - throw new IllegalArgumentException( - "For CUSTOM_PARAMS line " + paramLine + " can not be parsed. This field should contain only lines like PARAM1=VALUE1"); - - - String valueOfParam = pairs[1].replaceAll("(\"|\\\\)", "\\\\$0"); - - if (valueOfParam.trim().startsWith("\'") && !valueOfParam.trim().startsWith("\'{") && !valueOfParam.trim().startsWith("\'[")) - throw new IllegalArgumentException( - "Key " + pairs[0] + " has incorrect value " + valueOfParam + " ' symbol is invalid for non-structured variable format"); - - k.put(pairs[0].trim(), new Parameter(processParam(pairs[1]), ParametersConstants.CUSTOM_PARAMS_ORIGIN, true)); - } - } - return k; - } -} - diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/service/ParametersCalculationServiceV1.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/service/ParametersCalculationServiceV1.java index f8b5bae94..af40cc438 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/service/ParametersCalculationServiceV1.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/service/ParametersCalculationServiceV1.java @@ -43,23 +43,19 @@ public ParametersCalculationServiceV1(ParametersProcessor parametersProcessor) { } public ParameterBundle getCliParameter(String tenantName, String cloudName, String namespaceName, String applicationName, - DeployerInputs deployerInputs, String originalNamespace, Map k8TokenMap) { - return getParameterBundle(tenantName, cloudName, namespaceName, applicationName, deployerInputs, originalNamespace, k8TokenMap); - } - - public ParameterBundle getCliE2EParameter(String tenantName, String cloudName) { - return getE2EParameterBundle(tenantName, cloudName); + DeployerInputs deployerInputs, String originalNamespace) { + return getParameterBundle(tenantName, cloudName, namespaceName, applicationName, deployerInputs, originalNamespace); } private ParameterBundle getParameterBundle(String tenantName, String cloudName, String namespaceName, String applicationName, DeployerInputs deployerInputs - , String originalNamespace, Map k8TokenMap) { + , String originalNamespace) { Params parameters = parametersProcessor.processAllParameters(tenantName, cloudName, namespaceName, applicationName, - "false", deployerInputs, - originalNamespace); + originalNamespace, + new HashMap<>()); ParameterBundle parameterBundle = ParameterBundle.builder().build(); @@ -68,13 +64,6 @@ private ParameterBundle getParameterBundle(String tenantName, String cloudName, return parameterBundle; } - private ParameterBundle getE2EParameterBundle(String tenantName, String cloudName) { - Params parameters = parametersProcessor.processE2EParameters(tenantName, cloudName, null, null, "false", null, null); - ParameterBundle parameterBundle = ParameterBundle.builder().build(); - prepareSecureInsecureParams(parameters.getE2eParams(), parameterBundle, ParameterType.E2E); - return parameterBundle; - } - public void prepareSecureInsecureParams(Map parameters, ParameterBundle parameterBundle, ParameterType parameterType) { Map securedParams = new TreeMap<>(); Map inSecuredParams = new TreeMap<>(); diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/service/ParametersCalculationServiceV2.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/service/ParametersCalculationServiceV2.java index 8992ee62d..34fd04841 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/service/ParametersCalculationServiceV2.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/service/ParametersCalculationServiceV2.java @@ -21,6 +21,7 @@ import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.ObjectUtils; +import org.qubership.cloud.devops.commons.pojo.parameterset.CustomParameterDTO; import org.qubership.cloud.devops.commons.utils.Parameter; import org.qubership.cloud.devops.commons.utils.ParameterUtils; import org.qubership.cloud.parameters.processor.ParametersProcessor; @@ -34,6 +35,7 @@ import java.nio.charset.StandardCharsets; import java.util.*; +import static org.qubership.cloud.devops.commons.utils.ParameterUtils.prepareCustomParams; import static org.qubership.cloud.devops.commons.utils.constant.ApplicationConstants.*; import static org.qubership.cloud.devops.commons.utils.constant.NamespaceConstants.SSL_SECRET; @@ -49,8 +51,11 @@ public ParametersCalculationServiceV2(ParametersProcessor parametersProcessor) { } public ParameterBundle getCliParameter(String tenantName, String cloudName, String namespaceName, String applicationName, - DeployerInputs deployerInputs, String originalNamespace, Map k8TokenMap) { - return getParameterBundle(tenantName, cloudName, namespaceName, applicationName, deployerInputs, originalNamespace, k8TokenMap); + DeployerInputs deployerInputs, String originalNamespace, + Map k8TokenMap, CustomParameterDTO customParams) { + return getParameterBundle(tenantName, cloudName, namespaceName, + applicationName, deployerInputs, originalNamespace, + k8TokenMap, customParams); } public ParameterBundle getCliE2EParameter(String tenantName, String cloudName) { @@ -63,7 +68,6 @@ public ParameterBundle getCleanupParameterBundle(String tenantName, String cloud Params parameters = parametersProcessor.processNamespaceParameters(tenantName, cloudName, namespaceName, - "false", deployerInputs, originalNamespace); @@ -74,14 +78,15 @@ public ParameterBundle getCleanupParameterBundle(String tenantName, String cloud } private ParameterBundle getParameterBundle(String tenantName, String cloudName, String namespaceName, String applicationName, - DeployerInputs deployerInputs, String originalNamespace, Map k8TokenMap) { + DeployerInputs deployerInputs, String originalNamespace, + Map k8TokenMap, CustomParameterDTO customParams) { Params parameters = parametersProcessor.processAllParameters(tenantName, cloudName, namespaceName, applicationName, - "false", deployerInputs, - originalNamespace); + originalNamespace, + customParams.getAllParams()); ParameterBundle parameterBundle = ParameterBundle.builder().build(); @@ -91,6 +96,11 @@ private ParameterBundle getParameterBundle(String tenantName, String cloudName, if (MapUtils.isNotEmpty(parameters.getDeployParams()) && parameters.getDeployParams().containsKey(DEPLOY_DESC)) { processDeploymentDescriptorParams(parameters, parameterBundle); } + if (MapUtils.isNotEmpty(customParams.getAllParams())) { + prepareCustomParams(customParams, parameters.getDeployParams(), parameters.getTechParams()); + parameterBundle.setCustomDeployParameters(ParametersProcessor.convertParameterMapToObject(customParams.getDeployParams())); + parameterBundle.setCustomTechParameters(ParametersProcessor.convertParameterMapToObject(customParams.getTechnicalParams())); + } prepareSecureInsecureParams(parameters.getDeployParams(), parameterBundle, ParameterType.DEPLOY, k8TokenMap, originalNamespace); prepareSecureInsecureParams(parameters.getTechParams(), parameterBundle, ParameterType.TECHNICAL, k8TokenMap, originalNamespace); return parameterBundle; @@ -158,7 +168,7 @@ private static void processDeploymentDescriptorParams(Params parameters, Paramet } private ParameterBundle getE2EParameterBundle(String tenantName, String cloudName) { - Params parameters = parametersProcessor.processE2EParameters(tenantName, cloudName, null, null, "false", null, null); + Params parameters = parametersProcessor.processE2EParameters(tenantName, cloudName, null, null, null, null); ParameterBundle parameterBundle = ParameterBundle.builder().build(); prepareSecureInsecureParams(parameters.getE2eParams(), parameterBundle, ParameterType.E2E, null, null); return parameterBundle; @@ -168,7 +178,7 @@ public void prepareSecureInsecureParams(Map parameters, Param , ParameterType parameterType, Map k8TokenMap, String originalNamespace) { Map securedParams = new TreeMap<>(); Map inSecuredParams = new TreeMap<>(); - if (parameters == null || parameters.isEmpty()) { + if (MapUtils.isEmpty(parameters) && MapUtils.isEmpty(parameterBundle.getCustomTechParameters())) { LOGGER.debug("No Parameters found. Check if the input values are correct"); return; } @@ -182,7 +192,7 @@ public void prepareSecureInsecureParams(Map parameters, Param } else if (parameterType == ParameterType.DEPLOY) { handleDeployParameters(parameterBundle, k8TokenMap, originalNamespace, finalSecuredParams, inSecuredParamsAsObject); } else if (parameterType == ParameterType.TECHNICAL) { - parameterBundle.setSecuredConfigParams(finalSecuredParams); + prepareCustomTechSecureParams(parameterBundle, finalSecuredParams); parameterBundle.setConfigServerParams(inSecuredParamsAsObject); } else if (parameterType == ParameterType.CLEANUP) { finalSecuredParams.put(K8S_TOKEN, k8TokenMap.get(originalNamespace)); @@ -191,6 +201,18 @@ public void prepareSecureInsecureParams(Map parameters, Param } } + private static void prepareCustomTechSecureParams(ParameterBundle parameterBundle, Map finalSecuredParams) { + if (MapUtils.isEmpty(parameterBundle.getCustomTechParameters())) { + return; + } + Map customTechParams = ParametersProcessor.convertParameterMapToObject(parameterBundle.getCustomTechParameters()); + if (MapUtils.isEmpty(finalSecuredParams)) { + parameterBundle.setSecuredConfigParams(new TreeMap<>(customTechParams)); + } else { + parameterBundle.getSecuredConfigParams().putAll(customTechParams); + } + } + private void handleDeployParameters(ParameterBundle parameterBundle, Map k8TokenMap, String originalNamespace, Map finalSecuredParams, Map inSecuredParamsAsObject) { Object appChartName = inSecuredParamsAsObject.get(APPR_CHART_NAME); parameterBundle.setAppChartName(appChartName != null ? appChartName.toString() : ""); diff --git a/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/BindingBaseTest.java b/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/BindingBaseTest.java index fa54812ab..c67b4c724 100644 --- a/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/BindingBaseTest.java +++ b/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/BindingBaseTest.java @@ -124,12 +124,11 @@ public T withSpan(String spanName, ThrowingSupplier constructor = Binding.class.getDeclaredConstructor(String.class); + Constructor constructor = Binding.class.getDeclaredConstructor(); constructor.setAccessible(true); - Binding binding = constructor.newInstance(params.get("defaultEscapeSequence") != null ? params.get("defaultEscapeSequence") : "false") - .init("tenant", "cloud", "namespace", "application", "namespace"); + Binding binding = constructor.newInstance() + .init("tenant", "cloud", "namespace", "application", "namespace", new HashMap<>()); return binding; - } catch (NoSuchMethodException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | SecurityException e) { throw new RuntimeException(e); diff --git a/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/expression/BindingTest.java b/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/expression/BindingTest.java index b3ae9a1d4..2cc347e94 100644 --- a/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/expression/BindingTest.java +++ b/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/expression/BindingTest.java @@ -56,19 +56,9 @@ public void init_ESCAPE_SEQUENCE_true() throws NoSuchMethodException, Instantiat }; Binding binding = setupBinding(params); - assertEquals("true", binding.getProcessorType()); assertTrue(binding.get("tenant").get("cloud").get("yaml").getValue() instanceof Map); } - @Test - public void init_ESCAPE_SEQUENCE_default() throws NoSuchMethodException, InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, SecurityException { - Map params = new HashMap(); - - Binding binding = setupBinding(params); - - assertEquals("false", binding.getProcessorType()); - } - @Test public void init_UsernamePassword() throws NoSuchMethodException, InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, SecurityException { diff --git a/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/expression/ExpressionLanguageTest.java b/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/expression/ExpressionLanguageTest.java index 37ed10c9e..89357e9b9 100644 --- a/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/expression/ExpressionLanguageTest.java +++ b/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/expression/ExpressionLanguageTest.java @@ -448,7 +448,7 @@ public void processBackslashes(String string, String result) throws IllegalAcces @ParameterizedTest @MethodSource public void processValue(Map params, Map result) throws IllegalAccessException, InvocationTargetException, NoSuchMethodException { - Binding binding = new Binding("true"); + Binding binding = new Binding(); binding.putAll(params); ExpressionLanguage el = new ExpressionLanguage(binding); @@ -548,7 +548,7 @@ public void processDeployment_check_return_value_is_not_Parameter() { @Test void processedGlobalResourceProfileMustBeSuccessfullyProcessedAgain() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { - Binding binding = new Binding("true"); + Binding binding = new Binding(); binding.setDefault(""); HashMap map = new HashMap<>(){{ put("key1", "value1"); @@ -566,4 +566,125 @@ void processedGlobalResourceProfileMustBeSuccessfullyProcessedAgain() throws NoS processMap.setAccessible(true); assertEquals("{GLOBAL_RESOURCE_PROFILE={key1=value1, key2=value2}}", processMap.invoke(el, binding, binding, true).toString()); } + + // Test that data types are preserved when one parameter references another. + // This addresses the issue where EXPVAR: ${ORGVAR} was being converted to a String instead of preserving the Integer type. + + @Test + void testTypePreservationForIntegerReference() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + Binding binding = new Binding(); + + Parameter intParam = new Parameter(27017); + binding.put("MONGO_DB_PORT", intParam); + + Parameter refParam = new Parameter("${MONGO_DB_PORT}"); + binding.put("DUMPS_MONGO_PORT", refParam); + + ExpressionLanguage el = new ExpressionLanguage(binding); + Method processValue = ExpressionLanguage.class.getDeclaredMethod("processValue", Object.class); + processValue.setAccessible(true); + + Parameter result = (Parameter) processValue.invoke(el, refParam); + + assertThat(result.getValue(), instanceOf(Integer.class)); + assertEquals(27017, result.getValue()); + } + + @Test + void testTypePreservationForBooleanWithBraceSyntax() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + Binding binding = new Binding(); + + Parameter boolParam = new Parameter(true); + binding.put("ENABLE_SSL", boolParam); + + Parameter refParam = new Parameter("${ENABLE_SSL}"); + + ExpressionLanguage el = new ExpressionLanguage(binding); + Method processValue = ExpressionLanguage.class.getDeclaredMethod("processValue", Object.class); + processValue.setAccessible(true); + + Parameter result = (Parameter) processValue.invoke(el, refParam); + + assertThat(result.getValue(), instanceOf(Boolean.class)); + assertEquals(true, result.getValue()); + } + + @Test + void testTypePreservationForBooleanWithDollarSyntax() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + Binding binding = new Binding(); + + Parameter boolParam = new Parameter(true); + binding.put("ENABLE_SSL", boolParam); + + Parameter refParam = new Parameter("$ENABLE_SSL"); + + ExpressionLanguage el = new ExpressionLanguage(binding); + Method processValue = ExpressionLanguage.class.getDeclaredMethod("processValue", Object.class); + processValue.setAccessible(true); + + Parameter result = (Parameter) processValue.invoke(el, refParam); + + assertThat(result.getValue(), instanceOf(Boolean.class)); + assertEquals(true, result.getValue()); + } + + @Test + void testTypePreservationForGroovyStyleReference() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + Binding binding = new Binding(); + + Parameter intParam = new Parameter(8080); + binding.put("BASE_PORT", intParam); + + Parameter refParam = new Parameter("$BASE_PORT"); + binding.put("SERVER_PORT", refParam); + + ExpressionLanguage el = new ExpressionLanguage(binding); + Method processValue = ExpressionLanguage.class.getDeclaredMethod("processValue", Object.class); + processValue.setAccessible(true); + + Parameter result = (Parameter) processValue.invoke(el, refParam); + + assertThat(result.getValue(), instanceOf(Integer.class)); + assertEquals(8080, result.getValue()); + } + + @Test + void testTypePreservationForLongReference() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + Binding binding = new Binding(); + + Parameter longParam = new Parameter(604800000L); + binding.put("CDC_TOPIC_STREAMING_RETENTION_MS", longParam); + + Parameter refParam = new Parameter("${CDC_TOPIC_STREAMING_RETENTION_MS}"); + binding.put("RETENTION_COPY", refParam); + + ExpressionLanguage el = new ExpressionLanguage(binding); + Method processValue = ExpressionLanguage.class.getDeclaredMethod("processValue", Object.class); + processValue.setAccessible(true); + + Parameter result = (Parameter) processValue.invoke(el, refParam); + + assertThat(result.getValue(), instanceOf(Long.class)); + assertEquals(604800000L, result.getValue()); + } + + @Test + void testStringReferenceShouldNotPreserveType() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + Binding binding = new Binding(); + + Parameter strParam = new Parameter("myhost.local"); + binding.put("TEST_HOST", strParam); + + Parameter refParam = new Parameter("${TEST_HOST}"); + + ExpressionLanguage el = new ExpressionLanguage(binding); + Method processValue = ExpressionLanguage.class.getDeclaredMethod("processValue", Object.class); + processValue.setAccessible(true); + + Parameter result = (Parameter) processValue.invoke(el, refParam); + + assertThat(result.getValue(), instanceOf(String.class)); + assertEquals("myhost.local", result.getValue()); + } + } diff --git a/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/expression/LanguageTest.java b/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/expression/LanguageTest.java index bf48f9417..76c33429c 100644 --- a/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/expression/LanguageTest.java +++ b/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/expression/LanguageTest.java @@ -25,7 +25,6 @@ import org.qubership.cloud.devops.commons.pojo.parameterset.ParameterSetApplication; import org.qubership.cloud.devops.commons.utils.Parameter; import org.qubership.cloud.parameters.processor.expression.ExpressionLanguage; -import org.qubership.cloud.parameters.processor.expression.PlainLanguage; import org.qubership.cloud.parameters.processor.expression.binding.Binding; import org.junit.jupiter.api.Test; @@ -97,11 +96,6 @@ private Binding prepareBinding() { }); } - @Test - public void processDeployment_Plain_validate_order() { - Binding binding = prepareBinding(); - assertMap(new PlainLanguage(binding).processDeployment()); - } @Test public void processDeployment_Expression_validate_order() { diff --git a/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/expression/PlainLanguageTest.java b/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/expression/PlainLanguageTest.java deleted file mode 100644 index b8528fb0d..000000000 --- a/build_effective_set_generator/parameters-processor/src/test/java/org/qubership/cloud/expression/PlainLanguageTest.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright 2024-2025 NetCracker Technology Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.qubership.cloud.expression; - -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.qubership.cloud.BindingBaseTest; -import org.qubership.cloud.devops.commons.utils.Parameter; -import org.qubership.cloud.parameters.processor.ParametersProcessor; -import org.qubership.cloud.parameters.processor.expression.PlainLanguage; -import org.qubership.cloud.parameters.processor.expression.binding.Binding; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.Map; -import java.util.stream.Stream; - -class PlainLanguageTest extends BindingBaseTest { - private static Stream processDeployment() { - return Stream.of( - Arguments.of(new HashMap(), - new HashMap(), - new HashMap()), - Arguments.of(new HashMap() {{ - put("key1", "value1"); - put("key2", "$key1"); - }}, - new HashMap(), - new HashMap() {{ - put("key1", "value1"); - put("key2", "$key1"); - }}), - Arguments.of(new HashMap() {{ - put("key1", "value1"); - put("key2", "${cloud.key1}"); - }}, - new HashMap() {{ - put("key1", "value2"); - }}, - new HashMap() {{ - put("key1", "value2"); - put("key2", "${cloud.key1}"); - }}), - Arguments.of(new HashMap() {{ - put("key1", "'{\"key\": \"value\"}'"); - put("key2", "'{\"key\": [\"value1\", \"value2\"]}'"); - }}, - new HashMap() {{ - put("key2", "'{\"key\": [\"value3\"]}'"); - }}, - new HashMap() {{ - put("key1", new HashMap() {{ - put("key", "value"); - }}); - put("key2", new HashMap() {{ - put("key", new LinkedList() {{ - add("value3"); - }}); - }}); - }}), - Arguments.of(new HashMap() {{ - put("key1", "\"value1\""); - }}, - new HashMap() {{ - put("key2", "\"value2\""); - }}, - new HashMap() {{ - put("key1", "\\\"value1\\\""); - put("key2", "\\\"value2\\\""); - }}) - ); - } - - - @ParameterizedTest - @MethodSource - void processDeployment(Map tenant, Map cloud, Map result) { - Binding binding = setupBinding(new HashMap() {{ - put("tenantParams", tenant); - put("cloudParams", cloud); - put("defaultEscapeSequence", "false"); - }}); - Map map = ParametersProcessor.convertParameterMapToObject(new PlainLanguage(binding).processDeployment()); - map.remove("MAAS_ENABLED"); - map.remove("VAULT_INTEGRATION"); - map.remove("NAMESPACE"); - map.remove("CLOUDNAME"); - map.remove("TENANTNAME"); - map.remove("APPLICATION_NAME"); - map.remove("PRODUCTION_MODE"); - map.remove("CLOUD_API_HOST"); - map.remove("CLOUD_PUBLIC_HOST"); - map.remove("CLOUD_PRIVATE_HOST"); - map.remove("CLOUD_PROTOCOL"); - map.remove("SERVER_HOSTNAME"); - map.remove("CUSTOM_HOST"); - map.remove("OPENSHIFT_SERVER"); - map.remove("CLOUD_API_PORT"); - assertEquals(result, map); - } - - @Test - void processE2E_check_return_value_is_not_Parameter() { - Binding binding = setupBinding(new HashMap() { - { - put("tenantParamsE2E", Collections.singletonMap("struct", "str")); - put("defaultEscapeSequence", "false"); - } - }); - Map map = new PlainLanguage(binding).processE2E(); - map.remove("MAAS_ENABLED"); - map.remove("VAULT_INTEGRATION"); - - assertThat(map.get("struct"), instanceOf(Parameter.class)); - } -} diff --git a/build_effective_set_generator/scripts/handle_effective_set_config.py b/build_effective_set_generator/scripts/handle_effective_set_config.py new file mode 100644 index 000000000..096ff2608 --- /dev/null +++ b/build_effective_set_generator/scripts/handle_effective_set_config.py @@ -0,0 +1,109 @@ +import json +import os +import tempfile +import shutil +import argparse +from envgenehelper import logger + +def handle_effective_set_config(config_str): + + if isinstance(config_str, str): + try: + config = json.loads(config_str) + except json.JSONDecodeError as e: + logger.error(f"Failed to parse JSON: {e}") + raise + + version = config.get("version") or "v2.0" + extra_args = [f"--effective-set-version={version}"] + + app_chart_value = config.get("app_chart_validation", True) + extra_args.append(f"--app_chart_validation={str(app_chart_value).lower()}") + + consumers = ( + config.get("contexts", {}) + .get("pipeline", {}) + .get("consumers", []) + ) + + if not isinstance(consumers, list) or not consumers: + logger.info("No consumers provided; skipping schema generation.") + result_args = { + "extra_args": extra_args, # Only --effective-set-version + } + logger.info(json.dumps(result_args)) + return result_args + + temp_root = tempfile.gettempdir() + schema_output_dir = os.path.join(temp_root, "schemas", "registered_consumer_specific") + os.makedirs(schema_output_dir, exist_ok=True) + logger.info(f"Ensured directory exists: {schema_output_dir}") + + image_schema_dir = "/module/schemas/registered_consumer_specific" + + for consumer in consumers: + schema_json = consumer.get("schema") + name = consumer.get("name") + consumer_version = consumer.get("version") + + if not name or not consumer_version: + logger.error(f"Consumer entry missing required 'name' or 'version'") + continue + + filename = f"{name}-{consumer_version}.schema.json" + schema_file_path = os.path.join(schema_output_dir, filename) + + if schema_json: + try: + logger.info(f"Processing schema file: {filename}") + with open(schema_file_path, 'w') as schema_file: + json.dump(schema_json, schema_file, indent=2) + if os.path.isfile(schema_file_path): + logger.info(f"Schema file written successfully: {schema_file_path}, size={os.path.getsize(schema_file_path)}") + else: + logger.error(f"Schema file NOT found after writing: {schema_file_path}") + logger.info(f"Wrote schema for consumer '{name}' to {schema_file_path}") + except Exception as e: + logger.error(f"Failed to write schema for consumer '{name}': {e}") + continue + + else: + fallback_path = os.path.join(image_schema_dir, filename) + if os.path.isfile(fallback_path): + try: + shutil.copy(fallback_path, schema_file_path) + logger.info(f"Copied schema for {name} to {schema_file_path}") + except Exception as e: + logger.error(f"Failed to copy schema: {e}") + continue + else: + logger.error(f"Schema not found: {fallback_path}") + continue + + extra_args.append(f"--pipeline-consumer-specific-schema-path={schema_file_path}") + + result_args = { + "extra_args": extra_args, + } + return result_args + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--effective-set-config", + required=True, + help="JSON string or path to JSON file" + ) + args = parser.parse_args() + config_str = args.effective_set_config + + logger.info(f"config_str inside: {config_str}") + + try: + result_args = handle_effective_set_config(config_str) + logger.info(f"Resolved Extra args: {result_args}") + with open("/tmp/effective_set_output.json", "w") as f: + json.dump(result_args, f) + except Exception as e: + logger.error(f"Error: {e}") + exit(1) \ No newline at end of file diff --git a/build_effective_set_generator/scripts/main.py b/build_effective_set_generator/scripts/main.py new file mode 100644 index 000000000..ce2669fa6 --- /dev/null +++ b/build_effective_set_generator/scripts/main.py @@ -0,0 +1,26 @@ +import click +from envgenehelper import encrypt_all_cred_files_for_env, decrypt_all_cred_files_for_env, validate_creds + + +@click.group(chain=True) +def crypt_manager(): + pass + + +@crypt_manager.command("decrypt_cred_files") +def decrypt_cred_files(): + decrypt_all_cred_files_for_env() + + +@crypt_manager.command("encrypt_cred_files") +def encrypt_cred_files(): + encrypt_all_cred_files_for_env() + + +@crypt_manager.command("validate_creds") +def validate_credentials(): + validate_creds() + + +if __name__ == "__main__": + crypt_manager() diff --git a/build_effective_set_generator/scripts/sboms_retention_policy.py b/build_effective_set_generator/scripts/sboms_retention_policy.py new file mode 100644 index 000000000..2cb1a4447 --- /dev/null +++ b/build_effective_set_generator/scripts/sboms_retention_policy.py @@ -0,0 +1,34 @@ +from envgenehelper import getenv_with_error, get_envgene_config_yaml, logger, cleanup_dir_by_size, deleteFileIfExists, \ + cleanup_dir_by_age, get_sboms_dir +from envgenehelper.constants import CI_JOB_ARTIFACT_MAX_SIZE_MB +from envgenehelper.models import SbomRetentionConfig + + +def sboms_retention_policy(): + work_dir = getenv_with_error('CI_PROJECT_DIR') + sboms_dir = get_sboms_dir(work_dir) + config = get_envgene_config_yaml() + sbom_retention = SbomRetentionConfig.model_validate(config.get("sbom_retention", {})) + + if not sbom_retention.enabled: + logger.info("SBOMs retention policy is disabled") + return + + if not sboms_dir.exists(): + logger.warning(f"There is no such directory: {sboms_dir}") + return + + logger.info("SBOMs retention policy is enabled") + for sbom_path in sboms_dir.iterdir(): + if sbom_path.is_file(): + logger.info(f"Removing outdated format file: {sbom_path}") + deleteFileIfExists(sbom_path) + + for app_sbom_dir in sboms_dir.iterdir(): + cleanup_dir_by_age(app_sbom_dir, sbom_retention.keep_versions_per_app) + + cleanup_dir_by_size(sboms_dir, CI_JOB_ARTIFACT_MAX_SIZE_MB) + + +if __name__ == "__main__": + sboms_retention_policy() diff --git a/build_effective_set_generator/scripts/test_sboms_retention_policy.py b/build_effective_set_generator/scripts/test_sboms_retention_policy.py new file mode 100644 index 000000000..e2da85d8b --- /dev/null +++ b/build_effective_set_generator/scripts/test_sboms_retention_policy.py @@ -0,0 +1,72 @@ +import time + +import pytest +from envgenehelper import cleanup_dir_by_age, cleanup_dir_by_size +from envgenehelper.test_helpers import TestHelpers + + +class TestSBOMSRetentionPolicy: + + @pytest.mark.unit + def test_cleanup_dir_by_age_removes_old_files(self, tmp_path): + now = time.time() + files = [ + tmp_path / "old.json", + tmp_path / "mid.json", + tmp_path / "new.json", + ] + TestHelpers.create_file(files[0], mtime=now - 300) + TestHelpers.create_file(files[1], mtime=now - 200) + TestHelpers.create_file(files[2], mtime=now - 100) + + cleanup_dir_by_age(tmp_path, keep_last=2) + + remaining = {f.name for f in tmp_path.iterdir()} + assert remaining == {"mid.json", "new.json"} + + @pytest.mark.unit + def test_cleanup_dir_by_age_keep_all(self, tmp_path): + now = time.time() + files = [ + tmp_path / "file1.json", + tmp_path / "file2.json", + tmp_path / "file3.json", + ] + TestHelpers.create_file(files[0], mtime=now - 180) + TestHelpers.create_file(files[1], mtime=now - 120) + TestHelpers.create_file(files[2], mtime=now - 60) + + cleanup_dir_by_age(tmp_path, keep_last=3) + + remaining = {f.name for f in tmp_path.iterdir()} + assert remaining == {"file1.json", "file2.json", "file3.json"} + + @pytest.mark.unit + def test_cleanup_dir_by_size_within_limit(self, tmp_path): + files = [ + tmp_path / "file1.json", + tmp_path / "file2.json", + tmp_path / "file3.json", + ] + for f in files: + TestHelpers.create_file(f, size=1024) + + cleanup_dir_by_size(tmp_path, max_size_mb=10) + + remaining = {f.name for f in tmp_path.iterdir()} + assert remaining == {"file1.json", "file2.json", "file3.json"} + + @pytest.mark.unit + def test_cleanup_dir_by_size_exceeds(self, tmp_path): + files = [ + tmp_path / "file1.json", + tmp_path / "file2.json", + tmp_path / "file3.json", + ] + for f in files: + TestHelpers.create_file(f, size=1024 * 1024) + + cleanup_dir_by_size(tmp_path, max_size_mb=2) + + remaining = {f.name for f in tmp_path.iterdir()} + assert remaining == set() diff --git a/build_envgene/ansible/ansible.cfg b/build_envgene/ansible/ansible.cfg deleted file mode 100644 index d0c087556..000000000 --- a/build_envgene/ansible/ansible.cfg +++ /dev/null @@ -1,21 +0,0 @@ -[defaults] -callbacks_enabled = ansible.posix.timer,ansible.posix.profile_tasks -force_color = 1 -host_key_checking = False -local_tmp = /module/ansible/tmp -retry_files_enabled = False -roles_path = /module/ansible/roles -collections_paths = /module/ansible/collections -stdout_callback = ansible.posix.debug -timeout = 300 -filter_plugins = /module/ansible/filter_plugins - - -[galaxy] -cache_dir=/module/ansible/galaxy_cache -token_path=/module/ansible/galaxy_token - -[ssh_connection] -pipelining = true -retries = 7 -ssh_args = -o ControlMaster=auto -oConnectTimeout=30 -o ControlPersist=60s -C -o PreferredAuthentications=publickey,password diff --git a/build_envgene/ansible/playbooks/git_commit.yaml b/build_envgene/ansible/playbooks/git_commit.yaml deleted file mode 100644 index 99ee59051..000000000 --- a/build_envgene/ansible/playbooks/git_commit.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Commit and push changes to git - hosts: localhost - connection: local - gather_facts: false - roles: - - role: git_commit diff --git a/build_envgene/ansible/roles/git_commit/defaults/main.yaml b/build_envgene/ansible/roles/git_commit/defaults/main.yaml deleted file mode 100644 index 6cf3014fd..000000000 --- a/build_envgene/ansible/roles/git_commit/defaults/main.yaml +++ /dev/null @@ -1,18 +0,0 @@ -### default paths -artifact_dest: /tmp/artifact.zip -build_env_path: /build_env -envgen_debug: "{{ lookup('env', 'envgen_debug') }}" - -env_name: "{{ lookup('env', 'ENV_NAME') }}" -cluster_name: "{{ lookup('env', 'CLUSTER_NAME') }}" -environment_name: "{{ lookup('env', 'ENVIRONMENT_NAME') }}" -base_dir: "{{ lookup('env', 'CI_PROJECT_DIR') }}" -env_template_vers: "{{ lookup('env', 'ENV_TEMPLATE_VERSION') }}" - -### configuration files -envs_directory_path: "{{ base_dir }}/environments" -registry_config_path: "{{ base_dir }}/configuration/registry.yml" -cred_config_path: "{{ base_dir }}/configuration/credentials/credentials.yml" - -instance_secret_key: "{{ lookup('env', 'SECRET_KEY') }}" -COMMIT_ENV: "{{ lookup('env', 'COMMIT_ENV)') }}" diff --git a/build_envgene/ansible/roles/git_commit/tasks/01_prepare_vars.yaml b/build_envgene/ansible/roles/git_commit/tasks/01_prepare_vars.yaml deleted file mode 100644 index b334cdfb8..000000000 --- a/build_envgene/ansible/roles/git_commit/tasks/01_prepare_vars.yaml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- name: Set env_definition path - set_fact: - env_definition_path: "{{ envs_directory_path + '/' + cluster_name + '/' + environment_name }}" diff --git a/build_envgene/ansible/roles/git_commit/tasks/02_git_commit.yaml b/build_envgene/ansible/roles/git_commit/tasks/02_git_commit.yaml deleted file mode 100644 index 23fd34fc3..000000000 --- a/build_envgene/ansible/roles/git_commit/tasks/02_git_commit.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -### script arguments -- name: Set script arguments - set_fact: - script_args: "--env_definition_path={{ env_definition_path }} --version_to_add={{ env_template_vers }}" - -- name: 03.1 Commit and push changes to git - shell: | - cd ${CI_PROJECT_DIR} - . /module/scripts/git_commit.sh diff --git a/build_envgene/ansible/roles/git_commit/tasks/main.yaml b/build_envgene/ansible/roles/git_commit/tasks/main.yaml deleted file mode 100644 index 0e8145c34..000000000 --- a/build_envgene/ansible/roles/git_commit/tasks/main.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: 01. prepare vars - include_tasks: 01_prepare_vars.yaml - -- name: 02. git commit - include_tasks: 02_git_commit.yaml diff --git a/build_envgene/build/Dockerfile b/build_envgene/build/Dockerfile index 6857fc417..5214f577d 100644 --- a/build_envgene/build/Dockerfile +++ b/build_envgene/build/Dockerfile @@ -1,171 +1,70 @@ # checkov:skip=CKV_DOCKER_3:This build container requires root privileges for CI/CD operations +# checkov:skip=CKV_DOCKER_8:Root required for workspace write (shutil.copy2 utime) in CI/CD +# checkov:skip=CKV2_DOCKER_1:sudo is required for certain operations in scripts -######################################### +# ═══════════════════════════════════════════════════════════════════════════════════ # Stage 1: Build -# Multi-stage build to reduce final image size -FROM python:3.12-alpine3.19 AS build - -# Install build dependencies -# checkov:skip=CKV2_DOCKER_1:sudo is required for certain operations in scripts -# hadolint ignore=DL3018 -RUN apk add --no-cache \ - gcc \ - musl-dev \ - libffi-dev \ - openssl-dev \ - libxml2-dev \ - libxslt-dev \ - zlib-dev \ - git \ - curl \ - jq \ - openssh-client \ - sudo \ - zip \ - unzip +# ═══════════════════════════════════════════════════════════════════════════════════ +FROM ghcr.io/netcracker/qubership-envgene-base-modules:1.0.0 AS build -# Copy configuration files COPY build_envgene/build/pip.conf /etc/pip.conf COPY build_envgene/build/requirements.txt /build/requirements.txt -COPY build_envgene/build/requirements.yml /build/requirements.yml COPY build_envgene/build/constraint.txt /build/constraint.txt COPY creds_rotation/build/requirements.txt /build/creds_rotation_requirements.txt -# Copy source code COPY python /python -COPY build_envgene/ansible /module/ansible COPY build_envgene/scripts /module/scripts COPY scripts/bg_manage /scripts/bg_manage COPY creds_rotation/scripts /module/creds_rotation_scripts -COPY build_* create_* produce_* sort* /build_env/ COPY scripts/build_env /build_env/scripts/build_env/ COPY scripts/build_template /build_env/scripts/build_template/ COPY scripts/cloud_passport/ /cloud_passport/scripts/ COPY schemas /build_env/schemas COPY scripts/utils /module/scripts/utils -ENV ANSIBLE_LIBRARY=/module/ansible/library - -# Create virtual environment and install Python packages -RUN python -m venv /module/venv -RUN /module/venv/bin/pip install --upgrade pip setuptools wheel -RUN /module/venv/bin/pip install --no-cache-dir --retries 10 --timeout 60 -r /build/requirements.txt -# Install essential Ansible collections -# Install to virtual environment site-packages for Python module access -RUN /module/venv/bin/ansible-galaxy collection install ansible.utils -p /module/venv/lib/python3.12/site-packages/ansible_collections -RUN /module/venv/bin/ansible-galaxy collection install ansible.posix -p /module/venv/lib/python3.12/site-packages/ansible_collections -RUN /module/venv/bin/ansible-galaxy collection install community.general -p /module/venv/lib/python3.12/site-packages/ansible_collections -# Also install to custom location for playbook usage -RUN /module/venv/bin/ansible-galaxy collection install ansible.utils -p /module/ansible/collections -RUN /module/venv/bin/ansible-galaxy collection install ansible.posix -p /module/ansible/collections -RUN /module/venv/bin/ansible-galaxy collection install community.general -p /module/ansible/collections - -RUN /module/venv/bin/pip install /python/jschon-sort -RUN /module/venv/bin/pip install /python/envgene -RUN /module/venv/bin/pip install /python/integration -RUN /module/venv/bin/pip install /python/artifact-searcher -RUN /module/venv/bin/pip install --no-cache-dir --no-deps -r /build/creds_rotation_requirements.txt - -# Download and install SOPS for secrets management -RUN wget --tries=3 \ - https://github.com/mozilla/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 \ - -O /usr/local/bin/sops && \ - chmod +x /usr/local/bin/sops - -# Aggressive cleanup to reduce image size -RUN apk del gcc musl-dev libffi-dev openssl-dev libxml2-dev libxslt-dev zlib-dev -RUN rm -rf /var/cache/apk/* /tmp/* /var/tmp/* /root/.cache -# Remove unnecessary files from Python packages -RUN find /module/venv/lib/python3.12/site-packages -name '*.pyc' -delete -# Don't remove test directories as they might be needed by Ansible -RUN find /module/venv/lib/python3.12/site-packages -name '*.pyo' -delete -RUN find /module/venv/lib/python3.12/site-packages -name '__pycache__' -type d -exec rm -rf {} + 2>/dev/null || true -# Remove heavy Ansible collections that are not essential (but keep ansible.posix and ansible.utils) -RUN rm -rf /module/venv/lib/python3.12/site-packages/ansible_collections/amazon /module/venv/lib/python3.12/site-packages/ansible_collections/azure /module/venv/lib/python3.12/site-packages/ansible_collections/google /module/venv/lib/python3.12/site-packages/ansible_collections/kubernetes 2>/dev/null || true -# Remove test packages that are not needed in runtime (but keep Ansible test files) -RUN rm -rf /module/venv/lib/python3.12/site-packages/pytest* /module/venv/lib/python3.12/site-packages/_pytest* 2>/dev/null || true -RUN /module/venv/bin/pip cache purge - -# Verify collections are still accessible after cleanup -RUN /module/venv/bin/python -c "import ansible_collections.ansible.posix; print('ansible.posix collection still accessible after cleanup')" - -# Set permissions -RUN chmod 754 /module/scripts/* -RUN chmod 754 /module/creds_rotation_scripts/* - -######################################### +RUN pip install --no-cache-dir --retries 10 --timeout 60 \ + -r /build/requirements.txt \ + /python/jschon-sort \ + /python/envgene \ + /python/integration \ + /python/artifact-searcher && \ + pip install --no-cache-dir --no-deps \ + -r /build/creds_rotation_requirements.txt + +RUN rm -rf /var/cache/apk/* /tmp/* /var/tmp/* /root/.cache && \ + find /module/venv/lib/python3.12/site-packages -name '*.pyc' -delete && \ + find /module/venv/lib/python3.12/site-packages -name '*.pyo' -delete && \ + find /module/venv/lib/python3.12/site-packages -name '__pycache__' -type d -exec rm -rf {} + 2>/dev/null || true && \ + rm -rf /module/venv/lib/python3.12/site-packages/pytest* \ + /module/venv/lib/python3.12/site-packages/_pytest* 2>/dev/null || true && \ + /module/venv/bin/pip cache purge + +RUN chmod 754 /module/scripts/* && \ + chmod 754 /module/creds_rotation_scripts/* + +# ═══════════════════════════════════════════════════════════════════════════════════ # Stage 2: Runtime -# Lightweight runtime image with only essential dependencies -FROM python:3.12-alpine3.19 AS runtime - -# Install only essential runtime dependencies -# checkov:skip=CKV2_DOCKER_1:sudo is required for certain operations in scripts -# hadolint ignore=DL3018 -RUN apk add --no-cache \ - bash \ - ca-certificates \ - curl \ - jq \ - yq \ - gettext \ - age \ - git \ - openssh-client \ - sudo \ - zip \ - unzip \ - tar +# ═══════════════════════════════════════════════════════════════════════════════════ +FROM ghcr.io/netcracker/qubership-envgene-base-modules:1.0.0 AS runtime -# Copy everything from build stage COPY --from=build /module /module COPY --from=build /scripts/bg_manage /scripts/bg_manage -COPY --from=build /usr/local/bin/sops /usr/local/bin/sops COPY --from=build /build_env /build_env COPY --from=build /cloud_passport /cloud_passport COPY --from=build /python /python COPY --from=build /etc/pip.conf /etc/pip.conf -# Verify collections are accessible in runtime stage -RUN /module/venv/bin/python -c "import ansible_collections.ansible.posix; print('ansible.posix collection accessible in runtime')" - -# Set permissions -RUN chmod +x /usr/local/bin/sops - -# Create directories that might be needed for CI environments -# These directories are commonly used by GitHub Actions and GitLab CI -RUN mkdir -p /__w/_temp/_runner_file_commands && \ - mkdir -p /github/workspace && \ - mkdir -p /github/home && \ - mkdir -p /builds && \ - mkdir -p /cache && \ - chmod 777 /__w/_temp/_runner_file_commands && \ - chmod 777 /github/workspace && \ - chmod 777 /github/home && \ - chmod 777 /builds && \ - chmod 777 /cache - -# Final cleanup -RUN rm -rf /var/cache/apk/* /tmp/* /var/tmp/* /root/.cache -RUN find /module/venv/lib/python3.12/site-packages -name '*.pyc' -delete -# Don't remove test directories as they might be needed by Ansible -RUN /module/venv/bin/pip cache purge -# Keep pip for runtime compatibility, but remove setuptools and wheel -RUN rm -rf /module/venv/lib/python3.12/site-packages/setuptools* /module/venv/lib/python3.12/site-packages/wheel* 2>/dev/null || true - -# Set environment -ENV PATH=/module/venv/bin:$PATH \ - PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 \ - ANSIBLE_LIBRARY=/module/ansible/library \ - ANSIBLE_COLLECTIONS_PATH=/module/venv/lib/python3.12/site-packages/ansible_collections:/module/ansible/collections +RUN rm -rf /var/cache/apk/* /tmp/* /var/tmp/* /root/.cache && \ + find /module/venv/lib/python3.12/site-packages -name '*.pyc' -delete && \ + /module/venv/bin/pip cache purge -# Simple root-based container for CI/CD environments -# This container runs as root to avoid permission issues in CI/CD pipelines -WORKDIR /module/ansible +RUN chown -R ci:ci /module /build_env /cloud_passport /scripts -# Health check HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ CMD python -c "import sys; sys.exit(0)" || exit 1 -# Default command +# checkov:skip=CKV_DOCKER_8:Root required for workspace write in CI/CD +# hadolint ignore=DL3002 +USER root +WORKDIR /module/scripts CMD ["bash"] diff --git a/build_envgene/build/requirements.txt b/build_envgene/build/requirements.txt index daea4326f..0316e1d2b 100644 --- a/build_envgene/build/requirements.txt +++ b/build_envgene/build/requirements.txt @@ -9,23 +9,24 @@ lxml==4.9.3 ruamel.yaml==0.18.5 ruamel.yaml.clib==0.2.8 jschon==0.11.0 -jsonschema==4.19.1 +jsonschema==4.24.1 jmespath==1.0.1 semantic-version==2.10.0 termcolor==2.4.0 -ansible-core==2.17.12 cffi==1.16.0 -click==8.1.3 +click==8.1.7 deepmerge==2.0 GitPython==3.1.45 pydantic==2.10.6 +Jinja2==3.1.6 # Additional required packages platformdirs>=3.0.0 -ansible-runner==2.4.0 + +google-auth~=2.34.0 +qubership-pipelines-common-library==2.0.3 # Removed heavy packages: # - shyaml, yamale, prettytable (not essential) # - ruyaml (duplicate of ruamel.yaml) -# - diagrams (heavy with typed-ast dependency) -# - ansible-base (replaced with ansible-core) +# - diagrams (heavy with typed-ast dependency) \ No newline at end of file diff --git a/build_envgene/build/requirements.yml b/build_envgene/build/requirements.yml deleted file mode 100644 index 41925492f..000000000 --- a/build_envgene/build/requirements.yml +++ /dev/null @@ -1,5 +0,0 @@ -collections: - - name: community.general - version: 7.0.1 - - name: ansible.posix - version: 1.5.4 diff --git a/build_envgene/scripts/git_commit.sh b/build_envgene/scripts/git_commit.sh index 314a75330..8234e82a8 100755 --- a/build_envgene/scripts/git_commit.sh +++ b/build_envgene/scripts/git_commit.sh @@ -1,11 +1,11 @@ #!/bin/bash set -e -job=$1 + +echo "===== SCRIPT START: $(date '+%H:%M:%S') =====" + retries=0 exit_code=0 -pattern="^[A-Z]+-[0-9]+$" - if [ -n "${GITHUB_ACTIONS}" ]; then # Logic for GitHub PLATFORM="github" @@ -28,6 +28,10 @@ elif [ -n "${GITLAB_CI}" ]; then TOKEN="${GITLAB_TOKEN}" fi +if [ -z "${TOKEN}" ]; then + echo "No auth token was found. Please check!" + exit 1 +fi echo "Platform: ${PLATFORM}" echo "Server Protocol: ${SERVER_PROTOCOL}" @@ -37,11 +41,6 @@ echo "Branch/Ref Name: ${REF_NAME}" echo "User Email: ${USER_EMAIL}" echo "User Name: ${USER_NAME}" -if [ -z "${TOKEN}" ]; then - echo "No auth token was found. Please check!" - exit 1 -fi - echo "ENV_NAME=${ENV_NAME}" echo "CLUSTER_NAME=${CLUSTER_NAME}" echo "ENVIRONMENT_NAME=${ENVIRONMENT_NAME}" @@ -52,7 +51,6 @@ echo "DEPLOYMENT_SESSION_ID=${DEPLOY_SESSION_ID}" export ticket_id=${DEPLOYMENT_TICKET_ID} -# commit message if [ -z "${COMMIT_MESSAGE}" ]; then message="${ticket_id} [ci_skip] Update \"${CLUSTER_NAME}/${ENVIRONMENT_NAME}\" environment" else @@ -122,7 +120,7 @@ if [ -d environments ]; then done fi -#Copying cred files modified as part of cred rotation job. +# Copying cred files modified as part of cred rotation job. CREDS_FILE="environments/credfilestoupdate.yml" if [ -f "$CREDS_FILE" ]; then echo "Processing $CREDS_FILE for copying filtered creds..." @@ -174,8 +172,9 @@ echo "Adding remote: ${REMOTE_URL}" git remote add origin "${REMOTE_URL}" -echo "Pulling contents from GIT (branch: ${REF_NAME})" -git pull origin "${REF_NAME}" +echo "Fetching contents from GIT (branch: ${REF_NAME})" +git fetch --depth=1 origin ${REF_NAME} +git switch -C ${REF_NAME} origin/${REF_NAME} # moving back environments folder and committing @@ -225,8 +224,10 @@ if [ -e /tmp/configuration ]; then fi if [ -e /tmp/sboms ]; then - echo "Restoring config folder" - cp -r /tmp/sboms . + echo "Restoring sboms folder" + rm -rf sboms + mkdir -p sboms + cp -r /tmp/sboms/. sboms/ fi if [ -e /tmp/gitlab-ci ]; then @@ -299,17 +300,17 @@ if [ "$exit_code" -ne 0 ]; then echo "Waiting ${sleep_time} seconds before retry..." sleep $sleep_time - echo "Pulling latest changes from origin/${REF_NAME}..." - git pull origin "${REF_NAME}" - pull_exit_code=$? + echo "Fetching latest changes from origin/${REF_NAME}..." + git fetch --depth=1 origin "${REF_NAME}" + fetch_exit_code=$? - if [ "$pull_exit_code" -ne 0 ]; then - echo "⚠ Pull failed with exit code: $pull_exit_code, continuing to next retry..." - continue + if [ "$fetch_exit_code" -ne 0 ]; then + echo "⚠ Fetch failed with exit code: $fetch_exit_code" + break fi - echo "Successfully pulled changes. Remote is now at: $(git rev-parse origin/${REF_NAME})" - echo "Local HEAD is at: $(git rev-parse HEAD)" + git reset --soft origin/"${REF_NAME}" + git commit -m "${message}" echo "Attempting push (retry $retries)..." git push origin HEAD:"${REF_NAME}" @@ -332,4 +333,6 @@ if [ "$exit_code" -ne 0 ]; then fi fi +echo "===== SCRIPT END: $(date '+%H:%M:%S') =====" + exit $exit_code diff --git a/build_envgene/scripts/prepare.sh b/build_envgene/scripts/prepare.sh deleted file mode 100755 index 7eb90cb44..000000000 --- a/build_envgene/scripts/prepare.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -set -e - -#### input variables -# envgen_debug -# envgen_args -# module_ansible_dir -# module_ansible_cfg -# module_inventory -# CI_SERVER_URL -# GITLAB_TOKEN - -playbook_name=$1 -ansible_dir=${module_ansible_dir} - -if ${envgen_debug} ; then set -o xtrace; fi - -chmod 700 "$ansible_dir" -cd "$ansible_dir" - -export ANSIBLE_CONFIG=${module_ansible_cfg} - -#### Run ansible -echo "ansible-playbook playbooks/$playbook_name -i ${module_inventory} ${envgen_args}" -if ansible-playbook "playbooks/$playbook_name" -i "${module_inventory}" ${envgen_args}; then - status=0 -else - status=$? -fi - -mkdir -p "$CI_PROJECT_DIR/build_env/tmp" -if [ -d "/build_env/tmp" ]; then - cp -r /build_env/tmp/* "$CI_PROJECT_DIR/build_env/tmp/" || true -fi - -exit $status - diff --git a/build_envgene/scripts/update_ca_cert.sh b/build_envgene/scripts/update_ca_cert.sh deleted file mode 100755 index 06cbd44f7..000000000 --- a/build_envgene/scripts/update_ca_cert.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -CA_FILE="$1" - -function getLinuxDisto { - if [[ -f /etc/os-release ]]; then - # freedesktop.org and systemd - . /etc/os-release - DIST=$NAME - elif type lsb_release >/dev/null 2>&1; then - # linuxbase.org - DIST=$(lsb_release -si) - elif [[ -f /etc/lsb-release ]]; then - # For some versions of Debian/Ubuntu without lsb_release command - . /etc/lsb-release - DIST=$DISTRIB_ID - elif [[ -f /etc/debian_version ]]; then - # Older Debian/Ubuntu/etc. - DIST=Debian - else - # Fall back to uname, e.g. "Linux ", also works for BSD, etc. - DIST=$(uname -s) - fi - # convert to lowercase - DIST="$(tr '[:upper:]' '[:lower:]' <<< "$DIST")" -} - -function updateCertificates { - if [[ -e "${CA_FILE}" && ! -z "${CA_FILE}" ]]; then - getLinuxDisto - echo "Linux Distribution identified as: $DIST" - if [[ "${DIST}" == *"debian"* || "${DIST}" == *"ubuntu"* ]]; then - cp "${CA_FILE}" /usr/local/share/ca-certificates/ - update-ca-certificates --fresh - echo "certs from ${CA_FILE} added to trusted root" - export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt #https://ubuntu.com/server/docs/install-a-root-ca-certificate-in-the-trust-store - elif [[ "${DIST}" == *"centos"* ]]; then - cp "${CA_FILE}" /etc/pki/ca-trust/source/anchors/ca.crt - update-ca-trust - echo "certs from ${CA_FILE} added to trusted root" - export REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt #https://techjourney.net/update-add-ca-certificates-bundle-in-redhat-centos/ - elif [[ "${DIST}" == *"alpine"* ]]; then - cat "${CA_FILE}" >> /etc/ssl/certs/ca-certificates.crt - echo "certs from ${CA_FILE} added to trusted root" - export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt #we copy the certs to this file in line 43 - elif [[ "${DIST}" == *"red hat"* ]]; then - mkdir -p /etc/pki/ca-trust/source/anchors - cp "${CA_FILE}" /etc/pki/ca-trust/source/anchors/ - update-ca-trust - echo "certs from ${CA_FILE} added to trusted root" - export REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt #https://www.redhat.com/en/blog/configure-ca-trust-list - fi - else - echo "CA file ${CA_FILE} not found or empty" - exit 1 - fi - echo "export REQUESTS_CA_BUNDLE=${REQUESTS_CA_BUNDLE}" >> ~/.bashrc -} - -updateCertificates diff --git a/build_pipegene/build/Dockerfile b/build_pipegene/build/Dockerfile index cf2c2c19a..149edca62 100644 --- a/build_pipegene/build/Dockerfile +++ b/build_pipegene/build/Dockerfile @@ -1,34 +1,13 @@ -######################################### +# ═══════════════════════════════════════════════════════════════════════════════════ # Stage 1: Build -# Multi-stage build to reduce final image size -FROM python:3.12-alpine3.19 AS build -SHELL ["/bin/ash", "-eo", "pipefail", "-c"] +# ═══════════════════════════════════════════════════════════════════════════════════ +FROM ghcr.io/netcracker/qubership-envgene-base-modules:1.0.0 AS build -# Install build dependencies (pinned) -RUN set -eux; \ - apk update; \ - ver() { apk list --verbose --available "$1" | head -n1 | sed -E "s/^$1-//; s/ .*//"; }; \ - apk add --no-cache \ - gcc="$(ver gcc)" \ - musl-dev="$(ver musl-dev)" \ - libffi-dev="$(ver libffi-dev)" \ - openssl-dev="$(ver openssl-dev)" \ - git="$(ver git)" \ - curl="$(ver curl)" \ - jq="$(ver jq)" \ - yq="$(ver yq)" \ - gettext="$(ver gettext)" \ - sed="$(ver sed)" \ - age="$(ver age)"; \ - rm -rf /var/cache/apk/* - -# Copy configuration files (merged into build directory) -COPY build_pipegene/build/sources.list /etc/apk/repositories +# Copy configuration files and source code COPY build_pipegene/build/pip.conf /etc/pip.conf COPY build_pipegene/build/constraint.txt /build/constraint.txt COPY build_pipegene/build/requirements.txt /build/requirements.txt -# Copy source code COPY python /python COPY schemas /module/schemas COPY base_modules/scripts /module/scripts @@ -37,19 +16,13 @@ COPY scripts/utils /module/scripts/scripts/utils/ COPY build_pipegene/pipegene_plugins /module/scripts/pipegene_plugins # Create virtual environment and install Python packages -RUN python -m venv /module/venv && \ - /module/venv/bin/pip install --upgrade pip setuptools wheel && \ - /module/venv/bin/pip install --no-cache-dir --retries 10 --timeout 60 -r /build/requirements.txt && \ - /module/venv/bin/pip install /python/integration /python/jschon-sort /python/envgene && \ - /module/venv/bin/pip install PyYAML - -# Download and install SOPS for secrets management (use curl for BusyBox compatibility) -RUN curl --fail --show-error --location --retry 3 \ - https://github.com/mozilla/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 \ - -o /usr/local/bin/sops && \ - chmod +x /usr/local/bin/sops +RUN pip install --no-cache-dir --retries 10 --timeout 60 \ + -r /build/requirements.txt \ + /python/integration \ + /python/jschon-sort \ + /python/envgene -# Aggressive cleanup to reduce image size +# Cleanup to reduce image size RUN apk del gcc musl-dev libffi-dev openssl-dev && \ rm -rf /var/cache/apk/* /tmp/* /var/tmp/* /root/.cache && \ find /module/venv/lib/python3.12/site-packages -name '*.pyc' -delete && \ @@ -64,75 +37,27 @@ RUN apk del gcc musl-dev libffi-dev openssl-dev && \ ( /module/venv/bin/pip cache purge 2>/dev/null || true ) && \ rm -rf /module/venv/lib/python3.12/site-packages/pip* 2>/dev/null || true -# Set permissions RUN chmod 754 /module/scripts/* -######################################### +# ═══════════════════════════════════════════════════════════════════════════════════ # Stage 2: Runtime -# Lightweight runtime image with only essential dependencies -FROM python:3.12-alpine3.19 AS runtime -SHELL ["/bin/ash", "-eo", "pipefail", "-c"] - -# Install only essential runtime dependencies (pinned) -RUN set -eux; \ - apk update; \ - ver() { apk list --verbose --available "$1" | head -n1 | sed -E "s/^$1-//; s/ .*//"; }; \ - apk add --no-cache \ - bash="$(ver bash)" \ - ca-certificates="$(ver ca-certificates)" \ - curl="$(ver curl)" \ - jq="$(ver jq)" \ - yq="$(ver yq)" \ - gettext="$(ver gettext)" \ - sed="$(ver sed)" \ - age="$(ver age)" \ - git="$(ver git)"; \ - rm -rf /var/cache/apk/* +# ═══════════════════════════════════════════════════════════════════════════════════ +FROM ghcr.io/netcracker/qubership-envgene-base-modules:1.0.0 AS runtime -# Copy everything from build stage COPY --from=build /module /module -COPY --from=build /usr/local/bin/sops /usr/local/bin/sops COPY --from=build /python /python COPY --from=build /etc/pip.conf /etc/pip.conf -# Set permissions -RUN chmod +x /usr/local/bin/sops - -# Create directories that might be needed for CI environments -RUN mkdir -p /__w/_temp/_runner_file_commands && \ - mkdir -p /github/workspace && \ - mkdir -p /github/home && \ - mkdir -p /builds && \ - mkdir -p /cache && \ - chmod 777 /__w/_temp/_runner_file_commands && \ - chmod 777 /github/workspace && \ - chmod 777 /github/home && \ - chmod 777 /builds && \ - chmod 777 /cache - # Final cleanup RUN set -eux; \ rm -rf /var/cache/apk/* /tmp/* /var/tmp/* /root/.cache; \ find /module/venv/lib/python3.12/site-packages -name '*.pyc' -delete; \ - /module/venv/bin/pip cache purge 2>/dev/null || true; \ - # Keep pip for runtime compatibility, but remove setuptools and wheel - rm -rf /module/venv/lib/python3.12/site-packages/setuptools* /module/venv/lib/python3.12/site-packages/wheel* 2>/dev/null || true - -# Set environment -ENV PATH=/module/venv/bin:$PATH \ - PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 + /module/venv/bin/pip cache purge 2>/dev/null || true; -# Create user for security -RUN addgroup ci && adduser -D -h /module/ -s /bin/bash -G ci ci && \ - chown ci:ci -R /module - -# Health check HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ CMD python -c "import sys; sys.exit(0)" || exit 1 +RUN chown -R ci:ci /module USER ci:ci WORKDIR /module/scripts - -# Default command CMD ["bash"] diff --git a/build_pipegene/build/requirements.txt b/build_pipegene/build/requirements.txt index ea17898f7..7d6f7e08b 100644 --- a/build_pipegene/build/requirements.txt +++ b/build_pipegene/build/requirements.txt @@ -1,13 +1,13 @@ -boto3==1.29.3 -botocore==1.32.3 +boto3==1.39.4 +botocore==1.39.4 gcip==3.0.2 jmespath==1.0.1 packaging==23.2 pip>=23.0 -setuptools>=68.0 +setuptools>=68,<82 python-dateutil==2.8.2 PyYAML==6.0.1 -s3transfer==0.7.0 +s3transfer==0.13.1 setuptools-git-versioning==1.13.5 six==1.16.0 toml==0.10.2 @@ -18,7 +18,7 @@ ruamel.yaml==0.18.5 ruamel.yaml.clib==0.2.8 ruyaml==0.91.0 jschon==0.11.0 -jsonschema==4.19.1 +jsonschema==4.24.1 diagrams==0.23.3 attrs==23.2.0 referencing==0.33.0 diff --git a/build_pipegene/scripts/appregdef_render_job.py b/build_pipegene/scripts/appregdef_render_job.py index 1537b7df1..7b8b36e24 100644 --- a/build_pipegene/scripts/appregdef_render_job.py +++ b/build_pipegene/scripts/appregdef_render_job.py @@ -4,17 +4,15 @@ from pipeline_helper import job_instance -def prepare_appregdef_render_job(pipeline, is_template_test, env_template_version, full_env, environment_name, - cluster_name, group_id, artifact_id, artifact_url, tags): +def prepare_appregdef_render_job(pipeline, params, full_env, environment_name, cluster_name, group_id, artifact_id, + artifact_url): logger.info(f'Prepare appregdef render job for {full_env}') - script = [ - '/module/scripts/handle_certs.sh', - ] - if env_template_version and not is_template_test: + script = [] + if params.get('ENV_TEMPLATE_VERSION') and not params.get('IS_TEMPLATE_TEST'): script.append('python3 /build_env/scripts/build_env/env_template/set_template_version.py') - - script.append('cd /build_env; python3 /build_env/scripts/build_env/appregdef_render.py') + + script.append('python3 /build_env/scripts/build_env/appregdef_render.py') appregdef_render_params = { "name": f'app_reg_def_render.{full_env}', @@ -28,22 +26,14 @@ def prepare_appregdef_render_job(pipeline, is_template_test, env_template_versio "FULL_ENV_NAME": full_env, "CLUSTER_NAME": cluster_name, "ENVIRONMENT_NAME": environment_name, - "ENV_TEMPLATE_TEST": "true" if is_template_test else "false", - "ENV_TEMPLATE_VERSION": env_template_version, "INSTANCES_DIR": "${CI_PROJECT_DIR}/environments", "GROUP_ID": group_id, "ARTIFACT_ID": artifact_id, "ARTIFACT_URL": artifact_url, - "GITLAB_RUNNER_TAG_NAME": tags, } appregdef_render_job = job_instance(params=appregdef_render_params, vars=appregdef_render_vars) - - appregdef_render_job.artifacts.add_paths("${CI_PROJECT_DIR}/environments/{full_env}") - appregdef_render_job.artifacts.add_paths("${CI_PROJECT_DIR}/configuration") - appregdef_render_job.artifacts.add_paths("${CI_PROJECT_DIR}/tmp") appregdef_render_job.artifacts.when = WhenStatement.ALWAYS - pipeline.add_children(appregdef_render_job) - - return appregdef_render_job \ No newline at end of file + + return appregdef_render_job diff --git a/build_pipegene/scripts/bg_manage_job.py b/build_pipegene/scripts/bg_manage_job.py index 664d855e1..150d939d1 100644 --- a/build_pipegene/scripts/bg_manage_job.py +++ b/build_pipegene/scripts/bg_manage_job.py @@ -2,7 +2,7 @@ from envgenehelper import logger from pipeline_helper import job_instance -def prepare_bg_manage_job(pipeline, full_env, tags): +def prepare_bg_manage_job(pipeline, full_env): logger.info(f'prepare_bg manage job for {full_env}') job_params = { @@ -13,11 +13,9 @@ def prepare_bg_manage_job(pipeline, full_env, tags): } job_vars = { "FULL_ENV_NAME": full_env, - "GITLAB_RUNNER_TAG_NAME" : tags } job = job_instance(params=job_params, vars=job_vars) - job.artifacts.add_paths(f"$CI_PROJECT_DIR/environments/{full_env}") job.artifacts.when = WhenStatement.ALWAYS pipeline.add_children(job) return job diff --git a/build_pipegene/scripts/credential_rotation_job.py b/build_pipegene/scripts/credential_rotation_job.py index 4b40cc80c..ddda46c1e 100644 --- a/build_pipegene/scripts/credential_rotation_job.py +++ b/build_pipegene/scripts/credential_rotation_job.py @@ -2,14 +2,13 @@ from envgenehelper import logger from pipeline_helper import job_instance -def prepare_credential_rotation_job(pipeline, full_env, environment_name, cluster_name, tags): +def prepare_credential_rotation_job(pipeline, full_env, environment_name, cluster_name): logger.info(f'Prepare credential_rotation_job job for {full_env}.') credential_rotation_params = { "name": f'credential_rotation.{full_env}', "image": '${envgen_image}', "stage": 'credential_rotation', "script": [ - '/module/scripts/handle_certs.sh', "python3 /module/creds_rotation_scripts/creds_rotation_handler.py", ], } @@ -17,12 +16,8 @@ def prepare_credential_rotation_job(pipeline, full_env, environment_name, cluste credential_rotation_vars = { "CLUSTER_NAME": cluster_name, "ENV_NAME": environment_name, - "envgen_args": " -vv", - "envgen_debug": "true", - "GITLAB_RUNNER_TAG_NAME" : tags } credential_rotation_job = job_instance(params=credential_rotation_params, vars=credential_rotation_vars) - credential_rotation_job.artifacts.add_paths("${CI_PROJECT_DIR}/environments") credential_rotation_job.artifacts.add_paths("${CI_PROJECT_DIR}/affected-sensitive-parameters.yaml") credential_rotation_job.artifacts.when = WhenStatement.ALWAYS pipeline.add_children(credential_rotation_job) diff --git a/build_pipegene/scripts/effective_set_job.py b/build_pipegene/scripts/effective_set_job.py new file mode 100644 index 000000000..2db54fecf --- /dev/null +++ b/build_pipegene/scripts/effective_set_job.py @@ -0,0 +1,126 @@ +import json +from os import getenv, environ +from pathlib import Path + +from gcip import WhenStatement, Need + +from envgenehelper import logger, get_sboms_dir +from envgenehelper import cleanup_targets +from pipeline_helper import job_instance + + +def prepare_generate_effective_set_job(pipeline, full_env_name, env_name, cluster_name, params): + logger.info(f'Prepare generate-effective-set job for {full_env_name}') + logger.info(f'Cleanup_targets: {cleanup_targets}') + + app_reg_defs_job = params["APP_REG_DEFS_JOB"] + artifact_app_defs_path = params["APP_DEFS_PATH"] + artifact_reg_defs_path = params["REG_DEFS_PATH"] + sd_version = params["SD_VERSION"] + sd_data = params["SD_DATA"] + deployment_id = params["DEPLOYMENT_SESSION_ID"] + effective_set_config = params["EFFECTIVE_SET_CONFIG"] + if "CUSTOM_PARAMS" in params: + custom_params = params["CUSTOM_PARAMS"] + + is_local_app_def = artifact_app_defs_path and artifact_reg_defs_path and app_reg_defs_job + + base_dir = getenv('CI_PROJECT_DIR') + + sd_path = Path(f'{base_dir}/environments/{full_env_name}/Inventory/solution-descriptor/sd.yaml') + # TODO it is necessary to remove unnecessary calls, leave only script calls in such jobs! bad for gsf delivery + script = [ + #Overriding sd_path to pick the correct value for CI_PROJECT_DIR + f'base_env_path="$CI_PROJECT_DIR/environments/{full_env_name}";', + 'app_defs_path="$base_env_path/AppDefs";', + 'reg_defs_path="$base_env_path/RegDefs";', + 'sboms_path="$CI_PROJECT_DIR/sboms";', + 'sd_path="$base_env_path/Inventory/solution-descriptor/sd.yaml";', + # cert handling for java + 'mkdir -p ${CI_PROJECT_DIR}/configuration/certs/', + 'if [ -f /default_cert.pem ]; then cp /default_cert.pem "${CI_PROJECT_DIR}/configuration/certs/"; fi', + 'for cert in "${CI_PROJECT_DIR}/configuration/certs/*" ; do [ -f "$cert" ] && keytool -import -trustcacerts -alias "$(basename "$cert")" -file "$cert" -keystore /etc/ssl/certs/keystore.jks -storepass changeit -noprompt; done', + 'python3 /module/scripts/main.py decrypt_cred_files', + f'[ -n "$APP_REG_DEFS_JOB" ] && [ -n "$APP_DEFS_PATH" ] && mkdir -p $app_defs_path && cp -rf {artifact_app_defs_path}/* $app_defs_path', + f'[ -n "$APP_REG_DEFS_JOB" ] && [ -n "$REG_DEFS_PATH" ] && mkdir -p $reg_defs_path && cp -fr {artifact_reg_defs_path}/* $reg_defs_path', + 'python3 /module/scripts/main.py validate_creds', + 'python3 /module/scripts/sboms_retention_policy.py' + ] + + cmdb_cli_cmd_call = [ + f"/module/scripts/utils/entrypoint.sh --env-id={full_env_name}", + "--envs-path=$CI_PROJECT_DIR/environments", + f"--output=$CI_PROJECT_DIR/environments/{full_env_name}/effective-set" + ] + + effective_set_config_dict = {} + if effective_set_config: + effective_set_config_dict = json.loads(effective_set_config) + + effective_set_version = effective_set_config_dict.get("version") or "v2.0" + full_sd_exists = sd_path.is_file() + sd_data = bool(sd_data) or bool(sd_version) + + if not (full_sd_exists and sd_data) and effective_set_version.lower() == "v1.0": + raise ValueError("Feature generation effective set for pipeline and topology context is not supported for v1.0") + + if full_sd_exists or sd_data: + cmdb_cli_cmd_call.extend([ + "--registries=${CI_PROJECT_DIR}/configuration/registry.yml", + "--sboms-path=$sboms_path", + "--sd-path=$sd_path", + ]) + + logger.info(f'Prepare generate_effective_set job for {full_env_name}.') + if effective_set_config: + logger.info(f"EFFECTIVE_SET_CONFIG: {effective_set_config}") + script.extend([ + f"python3 /module/scripts/handle_effective_set_config.py --effective-set-config '{effective_set_config}'", + 'extra_args=$(jq -r \'.extra_args // [] | join(" ")\' /tmp/effective_set_output.json)', + ]) + cmdb_cli_cmd_call.extend(["$extra_args"]) + if deployment_id: + cmdb_cli_cmd_call.extend([f"--extra_params=DEPLOYMENT_SESSION_ID={deployment_id}"]) + + if custom_params: + logger.info(f"custom_params : {custom_params}") + cmdb_cli_cmd_call.extend([f"--custom-params='{custom_params}'"]) + script.append(" ".join(cmdb_cli_cmd_call)) + script.append('python3 /module/scripts/main.py encrypt_cred_files') + + generate_effective_set_params = { + "name": f'generate_effective_set.{full_env_name}', + "image": '${effective_set_generator_image}', + "stage": 'generate_effective_set', + "script": script + } + + generate_effective_set_vars = { + "CLUSTER_NAME": cluster_name, + "ENVIRONMENT_NAME": env_name, + "ENV_NAME": env_name, + "INSTANCES_DIR": "${CI_PROJECT_DIR}/environments", + "effective_set_generator_image": "$effective_set_generator_image", + "EXCLUDE_CLEANUP_TARGETS": " ".join(cleanup_targets) + } + + needs = [] + if is_local_app_def: + # gcip library doesn't allow to create a Need object that has the same pipeline as one it runs within. + # We need to specify pipeline because generated job will be ran in child pipeline + # To work around this we temporarily change value in environment and return it after creating the Need object + real_ci_pipe_id = getenv('CI_PIPELINE_ID', '') # currect pipeline, parent of future child pipeline + environ['CI_PIPELINE_ID'] = '0000000' + needs.append(Need(job=app_reg_defs_job, pipeline=real_ci_pipe_id, artifacts=True)) + environ['CI_PIPELINE_ID'] = real_ci_pipe_id + generate_effective_set_job = job_instance(params=generate_effective_set_params, needs=needs, + vars=generate_effective_set_vars) + + effective_set_expiry = effective_set_config_dict.get("effective_set_expiry") or "1 hour" + logger.info(f"effective set expiry value '{effective_set_expiry}'") + generate_effective_set_job.artifacts.expire_in = effective_set_expiry + + generate_effective_set_job.artifacts.when = WhenStatement.ALWAYS + pipeline.add_children(generate_effective_set_job) + + return generate_effective_set_job diff --git a/build_pipegene/scripts/env_build_jobs.py b/build_pipegene/scripts/env_build_jobs.py index 3dbe4f66e..369c08291 100644 --- a/build_pipegene/scripts/env_build_jobs.py +++ b/build_pipegene/scripts/env_build_jobs.py @@ -3,19 +3,18 @@ from pipeline_helper import job_instance -def prepare_env_build_job(pipeline, is_template_test, full_env, enviroment_name, cluster_name, group_id, artifact_id, tags): +def prepare_env_build_job(pipeline, is_template_test, full_env, enviroment_name, cluster_name, group_id, artifact_id): logger.info(f'prepare env_build job for {full_env}') script = [ - '/module/scripts/handle_certs.sh', + 'cd /build_env; python3 /build_env/scripts/build_env/main.py' ] - script.append('cd /build_env; python3 /build_env/scripts/build_env/main.py') if is_template_test: script.append('env_name=$(cat "$CI_PROJECT_DIR/set_variable.txt")') script.append( 'sed -i "s|\\\"envgeneNullValue\\\"|\\\"test_value\\\"|g" "$CI_PROJECT_DIR/environments/$env_name/Credentials/credentials.yml"') - + env_build_params = { "name": f'env_builder.{full_env}', "image": '${envgen_image}', @@ -30,102 +29,41 @@ def prepare_env_build_job(pipeline, is_template_test, full_env, enviroment_name, "ENVIRONMENT_NAME": enviroment_name, "GROUP_ID": group_id, "ARTIFACT_ID": artifact_id, - "ENV_TEMPLATE_TEST": "true" if is_template_test else "false", "INSTANCES_DIR": "${CI_PROJECT_DIR}/environments", - "envgen_image": "$envgen_image", - "envgen_args": " -vvv", - "envgen_debug": "true", - "module_config_default": "/module/templates/defaults.yaml", - "GITLAB_RUNNER_TAG_NAME": tags, } env_build_job = job_instance(params=env_build_params, vars=env_build_vars) if is_template_test: - env_build_job.artifacts.add_paths("${CI_PROJECT_DIR}/environments") env_build_job.artifacts.add_paths("${CI_PROJECT_DIR}/set_variable.txt") - else: - env_build_job.artifacts.add_paths("${CI_PROJECT_DIR}/environments/" + f"{full_env}") - env_build_job.artifacts.add_paths("${CI_PROJECT_DIR}/configuration") - env_build_job.artifacts.add_paths("${CI_PROJECT_DIR}/tmp") + env_build_job.artifacts.when = WhenStatement.ALWAYS pipeline.add_children(env_build_job) return env_build_job -def prepare_generate_effective_set_job(pipeline, environment_name, cluster_name, tags): - logger.info(f'Prepare generate_effective_set job for {cluster_name}/{environment_name}.') - generate_effective_set_params = { - "name": f'generate_effective_set.{cluster_name}/{environment_name}', - "image": '${effective_set_generator_image}', - "stage": 'generate_effective_set', - "script": ['/module/scripts/prepare.sh "generate_effective_set.yaml"', - "export env_name=$(echo $ENV_NAME | awk -F '/' '{print $NF}')", - 'env_path=$(sudo find $CI_PROJECT_DIR/environments -type d -name "$env_name")', - 'for path in $env_path; do if [ -d "$path/Credentials" ]; then sudo chmod ugo+rw $path/Credentials/*; fi; done' - ], - } - - generate_effective_set_vars = { - "CLUSTER_NAME": cluster_name, - "ENVIRONMENT_NAME": environment_name, - "INSTANCES_DIR": "${CI_PROJECT_DIR}/environments", - "effective_set_generator_image": "$effective_set_generator_image", - "envgen_args": " -vv", - "envgen_debug": "true", - "module_ansible_dir": "/module/ansible", - "module_inventory": "${CI_PROJECT_DIR}/configuration/inventory.yaml", - "module_ansible_cfg": "/module/ansible/ansible.cfg", - "module_config_default": "/module/templates/defaults.yaml", - "GITLAB_RUNNER_TAG_NAME": tags - } - generate_effective_set_job = job_instance(params=generate_effective_set_params, vars=generate_effective_set_vars) - generate_effective_set_job.artifacts.add_paths( - "${CI_PROJECT_DIR}/environments/" + f"{cluster_name}/{environment_name}") - generate_effective_set_job.artifacts.when = WhenStatement.ALWAYS - pipeline.add_children(generate_effective_set_job) - return generate_effective_set_job - - -def prepare_git_commit_job(pipeline, full_env, enviroment_name, cluster_name, deployment_session_id, tags, - credential_rotation_job: object = None): +def prepare_git_commit_job(pipeline, full_env, enviroment_name, cluster_name, credential_rotation_job: object = None): logger.info(f'prepare git_commit job for {full_env}.') - logger.info(f'Deployment session id is {deployment_session_id}.') git_commit_params = { "name": f'git_commit.{full_env}', "image": '${envgen_image}', "stage": 'git_commit', "script": [ - '/module/scripts/handle_certs.sh', - '/module/scripts/prepare.sh "git_commit.yaml"', + '/module/scripts/git_commit.sh', "export env_name=$(echo $ENV_NAME | awk -F '/' '{print $NF}')", 'env_path=$(sudo find $CI_PROJECT_DIR/environments -type d -name "$env_name")', 'for path in $env_path; do if [ -d "$path/Credentials" ]; then sudo chmod ugo+rw $path/Credentials/*; fi; done', - 'cp -rf $CI_PROJECT_DIR/environments $CI_PROJECT_DIR/git_envs', - ], + ], } git_commit_vars = { "ENV_NAME": full_env, "CLUSTER_NAME": cluster_name, "ENVIRONMENT_NAME": enviroment_name, - "envgen_image": "$envgen_image", - "envgen_args": " -vv", - "envgen_debug": "true", - "module_ansible_dir": "/module/ansible", - "module_inventory": "${CI_PROJECT_DIR}/configuration/inventory.yaml", - "module_ansible_cfg": "/module/ansible/ansible.cfg", - "module_config_default": "/module/templates/defaults.yaml", - "GIT_STRATEGY": "none", "COMMIT_ENV": "true", - "GITLAB_RUNNER_TAG_NAME": tags, - "DEPLOY_SESSION_ID": deployment_session_id } git_commit_job = job_instance(params=git_commit_params, vars=git_commit_vars) - git_commit_job.artifacts.add_paths("${CI_PROJECT_DIR}/environments/" + f"{full_env}") - git_commit_job.artifacts.add_paths("${CI_PROJECT_DIR}/git_envs") - git_commit_job.artifacts.add_paths('${CI_PROJECT_DIR}/sboms') git_commit_job.artifacts.when = WhenStatement.ALWAYS if (credential_rotation_job is not None): git_commit_job.add_needs(credential_rotation_job) pipeline.add_children(git_commit_job) - return git_commit_job \ No newline at end of file + return git_commit_job diff --git a/build_pipegene/scripts/gitlab_ci.py b/build_pipegene/scripts/gitlab_ci.py index 5bd7290ca..866716c14 100644 --- a/build_pipegene/scripts/gitlab_ci.py +++ b/build_pipegene/scripts/gitlab_ci.py @@ -1,18 +1,22 @@ import os from os import listdir -from envgenehelper import logger, get_cluster_name_from_full_name, get_environment_name_from_full_name, parse_env_names +from envgenehelper import logger, get_cluster_name_from_full_name, get_environment_name_from_full_name from envgenehelper.plugin_engine import PluginEngine -from gcip import JobFilter, Pipeline +from gcip import JobFilter, Pipeline, TriggerJob import pipeline_helper from appregdef_render_job import prepare_appregdef_render_job from bg_manage_job import prepare_bg_manage_job from credential_rotation_job import prepare_credential_rotation_job -from env_build_jobs import prepare_env_build_job, prepare_generate_effective_set_job, prepare_git_commit_job +from env_build_jobs import prepare_env_build_job, prepare_git_commit_job from inventory_generation_job import prepare_inventory_generation_job, is_inventory_generation_needed from passport_jobs import prepare_trigger_passport_job, prepare_passport_job +from process_sd_job import prepare_process_sd +from effective_set_job import prepare_generate_effective_set_job from pipeline_helper import get_gav_coordinates_from_build, find_predecessor_job +from envgenehelper.collections_helper import split_multi_value_param + PROJECT_DIR = os.getenv('CI_PROJECT_DIR') or os.getenv('GITHUB_WORKSPACE') IS_GITLAB = bool(os.getenv('CI_PROJECT_DIR')) and not bool(os.getenv('GITHUB_ACTIONS')) @@ -21,9 +25,7 @@ logger.info(f"Detected environment - GitLab: {IS_GITLAB}, GitHub: {IS_GITHUB}") -def build_pipeline(params: dict) -> None: - tags = params['GITLAB_RUNNER_TAG_NAME'] - +def build_pipeline(params: dict, sensitive_params: list) -> None: artifact_url = None if params['IS_TEMPLATE_TEST']: logger.info("Generating jobs in template test mode.") @@ -54,7 +56,7 @@ def build_pipeline(params: dict) -> None: per_env_plugin_engine = PluginEngine(plugins_dir='/module/scripts/pipegene_plugins/per_env') - env_names = parse_env_names(params['ENV_NAMES']) + env_names = split_multi_value_param(params['ENV_NAMES']) if len(env_names) > 1 and is_inventory_generation_needed(params['IS_TEMPLATE_TEST'], params): raise ValueError( f"Generating Inventories for multiple Environments in single pipeline is not supported. " @@ -77,35 +79,34 @@ def build_pipeline(params: dict) -> None: environment_name = get_environment_name_from_full_name(full_env_name) job_sequence = [ - "bg_manage_job", "trigger_passport_job", "get_passport_job", + "bg_manage_job", "env_inventory_generation_job", "credential_rotation_job", "appregdef_render_job", + "process_sd_job", "env_build_job", "generate_effective_set_job", "git_commit_job" ] - if not params.get('BG_MANAGE', None): - logger.info(f'Preparing of bg_manage job for environment {full_env_name} is skipped.') - else: - jobs_map['bg_manage_job'] = prepare_bg_manage_job(pipeline, full_env_name, tags) - # get passport job if it is not already added for cluster if params['GET_PASSPORT'] and cluster_name not in get_passport_jobs: jobs_map["trigger_passport_job"] = prepare_trigger_passport_job(pipeline, full_env_name) - jobs_map["get_passport_job"] = prepare_passport_job(pipeline, full_env_name, - environment_name, cluster_name, tags) + jobs_map["get_passport_job"] = prepare_passport_job(pipeline, full_env_name, environment_name, cluster_name) get_passport_jobs[cluster_name] = True else: logger.info(f"Generation of cloud passport for environment '{full_env_name}' is skipped") + if not params.get('BG_MANAGE', None): + logger.info(f'Preparing of bg_manage job for environment {full_env_name} is skipped.') + else: + jobs_map['bg_manage_job'] = prepare_bg_manage_job(pipeline, full_env_name) + if is_inventory_generation_needed(params['IS_TEMPLATE_TEST'], params): jobs_map["env_inventory_generation_job"] = prepare_inventory_generation_job(pipeline, full_env_name, - environment_name, cluster_name, - params, tags) + environment_name, cluster_name) else: logger.info( f'Preparing of full_env_name inventory generation job for {full_env_name} ' @@ -114,38 +115,51 @@ def build_pipeline(params: dict) -> None: credential_rotation_job = None if params['CRED_ROTATION_PAYLOAD']: credential_rotation_job = prepare_credential_rotation_job(pipeline, full_env_name, environment_name, - cluster_name, tags) + cluster_name) jobs_map["credential_rotation_job"] = credential_rotation_job else: logger.info( f'Credential rotation job for {full_env_name} is skipped because CRED_ROTATION_PAYLOAD is empty.') if params['ENV_BUILD']: - jobs_map["appregdef_render_job"] = prepare_appregdef_render_job(pipeline, params['IS_TEMPLATE_TEST'], - params['ENV_TEMPLATE_VERSION'], - full_env_name, + jobs_map["appregdef_render_job"] = prepare_appregdef_render_job(pipeline, params, full_env_name, environment_name, cluster_name, group_id, - artifact_id, artifact_url, tags) + artifact_id, artifact_url) else: logger.info(f'Preparing of appregdef_render_job {full_env_name} is skipped.') + source_type = (params.get("SD_SOURCE_TYPE", "artifact")).lower() + if ( + (source_type == "json" and params.get("SD_DATA")) or + (source_type == "artifact" and params.get("SD_VERSION")) + ): + jobs_map["process_sd_job"] = prepare_process_sd(pipeline, full_env_name, environment_name, cluster_name) + else: + logger.info(f'Preparing of process_sd_job for {full_env_name} is skipped') + if params['ENV_BUILD']: jobs_map["env_build_job"] = prepare_env_build_job(pipeline, params['IS_TEMPLATE_TEST'], full_env_name, - environment_name, cluster_name, group_id, artifact_id, - tags) + environment_name, cluster_name, group_id, artifact_id) else: logger.info(f'Preparing of env_build job for {full_env_name} is skipped.') if params['GENERATE_EFFECTIVE_SET']: - jobs_map["generate_effective_set_job"] = prepare_generate_effective_set_job(pipeline, environment_name, - cluster_name, tags) + jobs_map["generate_effective_set_job"] = prepare_generate_effective_set_job(pipeline, full_env_name, + environment_name, cluster_name, + params) else: - logger.info(f'Preparing of generate_effective_set job for {cluster_name}/{environment_name} is skipped.') - - jobs_requiring_git_commit = ["appregdef_render_job", "env_build_job", "generate_effective_set_job", - "env_inventory_generation_job", "credential_rotation_job", "bg_manage_job"] - - plugin_params = params + logger.info(f'Preparing of generate_effective_set job for {full_env_name} is skipped.') + if "CUSTOM_PARAMS" in params and params["CUSTOM_PARAMS"]: + logger.warning( + "'CUSTOM_PARAMS' is only applied when ['GENERATE_EFFECTIVE_SET'](#generate_effective_set) " + "is 'true'. If 'GENERATE_EFFECTIVE_SET' is 'false', the 'generate_effective_set' job does not run " + "and 'CUSTOM_PARAMS' has no effect.") + + jobs_requiring_git_commit = ["appregdef_render_job", "process_sd_job", "env_build_job", + "generate_effective_set_job", "env_inventory_generation_job", + "credential_rotation_job", "bg_manage_job"] + + plugin_params = params.copy() plugin_params['jobs_map'] = jobs_map plugin_params['job_sequence'] = job_sequence plugin_params['jobs_requiring_git_commit'] = jobs_requiring_git_commit @@ -157,7 +171,6 @@ def build_pipeline(params: dict) -> None: if (any(job in jobs_map for job in plugin_params['jobs_requiring_git_commit']) and not params['IS_TEMPLATE_TEST']): jobs_map["git_commit_job"] = prepare_git_commit_job(pipeline, full_env_name, environment_name, cluster_name, - params['DEPLOYMENT_SESSION_ID'], tags, credential_rotation_job) else: logger.info(f'Preparing of git commit job for {full_env_name} is skipped.') @@ -173,13 +186,39 @@ def build_pipeline(params: dict) -> None: job_instance.add_needs(*find_predecessor_job(job, jobs_map, job_sequence)) logger.info(f'----------------end processing for {full_env_name}---------------------') + + for key, value in params.items(): + if key not in sensitive_params and value is not None and value != '': + sorted_pipeline.add_variables(**{key: value}) + sorted_pipeline.add_tags(params["GITLAB_RUNNER_TAG_NAME"]) # check out repo only once in the first job of the generated pipeline, later jobs get it through artifacts from each other # purpose: avoid later jobs restoring files that were removed by previous jobs, so git commit job can commit those deletions - for job in sorted_pipeline.find_jobs(JobFilter()): # gets all jobs in pipeline - job.artifacts.add_paths('./') - is_first_job = job.needs is None or len(job.needs) == 0 - if not is_first_job: - job.add_variables(GIT_CHECKOUT="false") + for job in sorted_pipeline.find_jobs(JobFilter()): + job.artifacts.add_paths( + 'environments/', + 'configuration/', + 'sboms/', + 'templates/', + 'tmp/' + ) + + if not do_checkout(job): + job.add_variables(GIT_STRATEGY="empty") sorted_pipeline.write_yaml() + +def is_trigger_job(job): + return isinstance(job, TriggerJob) + + +def do_checkout(job): + is_first_job = job.needs is None or len(job.needs) == 0 + if is_first_job or any(is_trigger_job(need) for need in job.needs): + logger.info( + f"Enabling checkout for {job.name} " + f"Stage: {job.stage}, Needs: {job.needs}" + ) + return True + + return False \ No newline at end of file diff --git a/build_pipegene/scripts/inventory_generation_job.py b/build_pipegene/scripts/inventory_generation_job.py index 8d59703ae..d3d3d30b7 100644 --- a/build_pipegene/scripts/inventory_generation_job.py +++ b/build_pipegene/scripts/inventory_generation_job.py @@ -10,7 +10,7 @@ def is_inventory_generation_needed(is_template_test, inventory_params): if is_template_test: return False - env_inventory_init = inventory_params.get('ENV_INVENTORY_INIT') == 'true' + env_inventory_init = inventory_params.get('ENV_INVENTORY_INIT') env_specific_parameters = inventory_params.get('ENV_SPECIFIC_PARAMS') env_template_name = inventory_params.get('ENV_TEMPLATE_NAME') env_inventory_content = inventory_params.get('ENV_INVENTORY_CONTENT') @@ -28,15 +28,13 @@ def is_inventory_generation_needed(is_template_test, inventory_params): return env_inventory_content or env_inventory_init or bool(env_specific_parameters) or bool(env_template_name) -def prepare_inventory_generation_job(pipeline, full_env_name, environment_name, cluster_name, env_generation_params, - tags): +def prepare_inventory_generation_job(pipeline, full_env_name, environment_name, cluster_name): logger.info(f"prepare env_generation job for {full_env_name}") params = { "name": f"env_inventory_generation.{full_env_name}", "image": "${envgen_image}", "stage": "env_inventory_generation", "script": [ - '/module/scripts/handle_certs.sh', "python3 /build_env/scripts/build_env/env_inventory_generation.py", ], } @@ -44,18 +42,8 @@ def prepare_inventory_generation_job(pipeline, full_env_name, environment_name, "ENV_NAME": environment_name, "CLUSTER_NAME": cluster_name, "FULL_ENV_NAME": full_env_name, - "envgen_image": "$envgen_image", - "envgen_args": " -vv", - "envgen_debug": "true", - "module_ansible_dir": "/module/ansible", - "module_inventory": "${CI_PROJECT_DIR}/configuration/inventory.yaml", - "module_ansible_cfg": "/module/ansible/ansible.cfg", - "module_config_default": "/module/templates/defaults.yaml", - "GITLAB_RUNNER_TAG_NAME": tags, - **env_generation_params } job = job_instance(params=params, vars=vars) - job.artifacts.add_paths("${CI_PROJECT_DIR}/environments/") job.artifacts.when = WhenStatement.ALWAYS pipeline.add_children(job) return job diff --git a/build_pipegene/scripts/main.py b/build_pipegene/scripts/main.py index e124be622..b4e5a10b1 100644 --- a/build_pipegene/scripts/main.py +++ b/build_pipegene/scripts/main.py @@ -16,7 +16,7 @@ def perform_generation(): handler = PipelineParametersHandler() handler.log_pipeline_params() validate_pipeline(handler.params) - build_pipeline(handler.params) + build_pipeline(handler.params, handler.sensitive_params) if __name__ == "__main__": gcip() diff --git a/build_pipegene/scripts/passport_jobs.py b/build_pipegene/scripts/passport_jobs.py index 0d6499eee..48ebca656 100644 --- a/build_pipegene/scripts/passport_jobs.py +++ b/build_pipegene/scripts/passport_jobs.py @@ -31,40 +31,30 @@ def prepare_trigger_passport_job(pipeline, full_env): return trigger_job -def prepare_passport_job(pipeline, full_env, enviroment_name, cluster_name, tags): +def prepare_passport_job(pipeline, full_env, enviroment_name, cluster_name): logger.info(f'prepare get_passport job for {full_env}') get_passport_params = { "name": f'get_passport.{full_env}', "image": '${envgen_image}', "stage": 'process_passport', - "script": ['/module/scripts/handle_certs.sh', - 'python3 /cloud_passport/scripts/main.py --env_name "$ENV_NAME",', - "export env_name=$(echo $ENV_NAME | awk -F '/' '{print $NF}')", - 'env_path=$(sudo find $CI_PROJECT_DIR/environments -type d -name "$env_name")', - 'for path in $env_path; do if [ -d "$path/Credentials" ]; then sudo chmod ugo+rw $path/Credentials/*; fi; done' - ], + "script": [ + 'python3 /cloud_passport/scripts/main.py --env_name "$ENV_NAME",', + "export env_name=$(echo $ENV_NAME | awk -F '/' '{print $NF}')", + 'env_path=$(sudo find $CI_PROJECT_DIR/environments -type d -name "$env_name")', + 'for path in $env_path; do if [ -d "$path/Credentials" ]; then sudo chmod ugo+rw $path/Credentials/*; fi; done' + ], } - get_passport_params['script'].append('/module/scripts/prepare.sh "git_commit.yaml"') + get_passport_params['script'].append('/module/scripts/git_commit.sh') get_passport_vars = { "ENV_NAME": full_env, "CLUSTER_NAME": cluster_name, "ENVIRONMENT_NAME": enviroment_name, - "envgen_image": "$envgen_image", - "envgen_args": " -vv", - "envgen_debug": "true", - "module_inventory": "${CI_PROJECT_DIR}/configuration/inventory.yaml", - "module_config_default": "/module/templates/defaults.yaml", - "COMMIT_ENV": "false", - "COMMIT_MESSAGE": f"[ci_skip] update cloud passport for {cluster_name}", - "GITLAB_RUNNER_TAG_NAME": tags, - "module_ansible_dir": "/module/ansible", - "module_ansible_cfg": "/module/ansible/ansible.cfg" } get_passport_job = job_instance(params=get_passport_params, vars=get_passport_vars) base = "${CI_PROJECT_DIR}/environments" - get_passport_job.artifacts.add_paths(f"{base}/{full_env}") get_passport_job.artifacts.add_paths(f"{base}/{cluster_name}/cloud-passport") get_passport_job.artifacts.when = WhenStatement.ALWAYS pipeline.add_children(get_passport_job) return get_passport_job + diff --git a/build_pipegene/scripts/pipeline_helper.py b/build_pipegene/scripts/pipeline_helper.py index e285b7d8f..6c7011dd0 100644 --- a/build_pipegene/scripts/pipeline_helper.py +++ b/build_pipegene/scripts/pipeline_helper.py @@ -32,7 +32,7 @@ def render(self) -> Dict[str, Any]: def job_instance(params, vars, needs=None, rules=None): timeout = getenv("RUNNER_SCRIPT_TIMEOUT") or "10m" - gitlab_runner_tag = vars.get('GITLAB_RUNNER_TAG_NAME') + job = JobExtended( name=params['name'], image=params['image'], @@ -45,7 +45,9 @@ def job_instance(params, vars, needs=None, rules=None): job.prepend_scripts(params['before_script']) global_before = [ - 'python /module/scripts/utils/log_pipe_params.py' + 'python /module/scripts/utils/log_pipe_params.py', + '/module/scripts/utils/handle_certs.sh', + 'source ~/.bashrc', ] job.prepend_scripts(*global_before) @@ -54,7 +56,7 @@ def job_instance(params, vars, needs=None, rules=None): if needs is None: needs = [] job.set_needs(needs) - job.add_tags(gitlab_runner_tag) + if rules: job.rules.extend(rules) return job diff --git a/build_pipegene/scripts/process_sd_job.py b/build_pipegene/scripts/process_sd_job.py new file mode 100644 index 000000000..641671f19 --- /dev/null +++ b/build_pipegene/scripts/process_sd_job.py @@ -0,0 +1,39 @@ +from os import getenv + +from gcip import WhenStatement + +from envgenehelper import logger +from pipeline_helper import job_instance + + +def prepare_process_sd(pipeline, full_env, environment_name, cluster_name): + logger.info(f'Prepare process_sd job for {full_env}') + + script = [ + f'base_env_path="$CI_PROJECT_DIR/environments/{full_env}";', + 'app_defs_path="$base_env_path/AppDefs";', + 'reg_defs_path="$base_env_path/RegDefs";', + f'[ -n "$APP_REG_DEFS_JOB" ] && [ -n "$APP_DEFS_PATH" ] && mkdir -p $app_defs_path && cp -rf $APP_DEFS_PATH/* $app_defs_path', + f'[ -n "$APP_REG_DEFS_JOB" ] && [ -n "$REG_DEFS_PATH" ] && mkdir -p $reg_defs_path && cp -fr $REG_DEFS_PATH/* $reg_defs_path', + 'python3 /build_env/scripts/build_env/process_sd.py', + ] + + process_sd_set_params = { + "name": f'process_sd.{full_env}', + "image": '${envgen_image}', + "stage": 'process_sd', + "script": script + } + + process_sd_set_vars = { + "CLUSTER_NAME": cluster_name, + "ENVIRONMENT_NAME": environment_name, + "ENV_NAME": environment_name, + "INSTANCES_DIR": "${CI_PROJECT_DIR}/environments", + } + + process_sd_job = job_instance(params=process_sd_set_params, vars=process_sd_set_vars) + process_sd_job.artifacts.when = WhenStatement.ALWAYS + pipeline.add_children(process_sd_job) + + return process_sd_job \ No newline at end of file diff --git a/build_pipegene/scripts/test_gitlab_ci.py b/build_pipegene/scripts/test_gitlab_ci.py index 7ce2301d4..28b551c97 100644 --- a/build_pipegene/scripts/test_gitlab_ci.py +++ b/build_pipegene/scripts/test_gitlab_ci.py @@ -1,12 +1,23 @@ -import pytest -from main import perform_generation -from envgenehelper import getAbsPath, openYaml, dump_as_yaml_format import os from dataclasses import dataclass, asdict +from pathlib import Path + +import pytest + +# validations / gitlab_ci capture CI_PROJECT_DIR at import time; set it before loading main. +_REPO_ROOT = Path(__file__).resolve().parents[2] +os.environ["CI_PROJECT_DIR"] = str(_REPO_ROOT / "test_data" / "pipegene_ci_instance") +os.environ["CI_JOB_NAME"] = 'JOB_NAME_PLACEHOLDER' +os.environ["CI_COMMIT_REF_SLUG"] = 'PLACEHOLDER' +os.environ.setdefault("JSON_SCHEMAS_DIR", str(_REPO_ROOT / "schemas")) + +from main import perform_generation +from envgenehelper import openYaml, dump_as_yaml_format + @dataclass class PipelineVars: - env_names: str = "sample-cloud-name/composite-full" + env_names: str = "cluster-01/env-01" env_template_version: str = "new-version:app_def" get_passport: str = "true" env_builder: str = "true" @@ -19,34 +30,53 @@ class PipelineVars: sd_version: str = "" env_template_name: str = "" env_specific_params: str = "" + custom_params: str = "" -def convert_keys_to_uppercase(dictionary): - return {k.upper(): v for k, v in dictionary} +def convert_keys_to_uppercase(pairs): + return {k.upper(): v for k, v in pairs} build_pipeline_test_data = [ - ( # with all jobs + ( PipelineVars(env_specific_params='{"params": "value"}'), - ["trigger", "process_passport", "env_inventory_generation", "env_builder", "generate_effective_set", "git_commit", "cmdb_import" ] + [ + "trigger", + "process_passport", + "env_inventory_generation", + "app_reg_def_render", + "env_builder", + "generate_effective_set", + "git_commit", + ], ), - ( # new version template test + ( PipelineVars(env_template_test="true", env_inventory_init="true"), - ["trigger", "process_passport", "env_builder", "generate_effective_set", "cmdb_import" ] + [ + "trigger", + "process_passport", + "app_reg_def_render", + "env_builder", + "generate_effective_set", + ], ), - ( # wihtout passport discovery + ( PipelineVars(get_passport="false"), - ["env_builder", "generate_effective_set", "git_commit", "cmdb_import" ] + ["app_reg_def_render", "env_builder", "generate_effective_set", "git_commit"], ), - ( # effective set only + ( PipelineVars(get_passport="false", env_builder="false", cmdb_import="false"), ["generate_effective_set", "git_commit"] ), - ( # without passport and effective set + ( PipelineVars(get_passport="false", generate_effective_set="false"), - ["env_builder", "git_commit", "cmdb_import" ] + ["app_reg_def_render", "env_builder", "git_commit"], ), - ( # with inventory generation and without env_build, passport discovery and effective set - PipelineVars(get_passport="false", env_builder="false", generate_effective_set="false", sd_data='{"params": "value"}'), - ["env_inventory_generation", "git_commit", "cmdb_import" ] + ( + PipelineVars(get_passport="false", custom_params='{"params": "value"}'), + ["app_reg_def_render", "env_builder", "generate_effective_set", "git_commit"], + ), + ( + PipelineVars(get_passport="false", generate_effective_set="false", custom_params='{"params": "value"}'), + ["app_reg_def_render", "env_builder", "git_commit"], ), ] @@ -56,9 +86,6 @@ def change_test_dir(request, monkeypatch): @pytest.mark.parametrize("pipeline_vars, expected_sequence", build_pipeline_test_data) def test_build_pipeline(pipeline_vars, expected_sequence): - os.environ["CI_PROJECT_DIR"] = getAbsPath("samples") - os.environ["JSON_SCHEMAS_DIR"] = getAbsPath("schemas") - ci_commit_ref_name = "feature/test-generate" os.environ["CI_COMMIT_REF_NAME"] = ci_commit_ref_name pipeline_vars = asdict(pipeline_vars, dict_factory=convert_keys_to_uppercase) diff --git a/build_pipegene/scripts/validations.py b/build_pipegene/scripts/validations.py index 582616bb5..bcbe85386 100644 --- a/build_pipegene/scripts/validations.py +++ b/build_pipegene/scripts/validations.py @@ -2,6 +2,7 @@ from os import getenv from envgenehelper import check_for_cyrillic, logger, findAllYamlsInDir, openYaml, check_dir_exists, get_cluster_name_from_full_name, get_environment_name_from_full_name, check_environment_is_valid_or_fail, check_file_exists, validate_yaml_by_scheme_or_fail +from envgenehelper.collections_helper import split_multi_value_param project_dir = os.getenv('CI_PROJECT_DIR') or os.getenv('GITHUB_WORKSPACE') logger.info(f"Info about project_dir: {project_dir}") @@ -42,7 +43,8 @@ def template_test_checks(): raise ReferenceError("Execution is aborted as validation is not successful. See logs above.") def real_execution_checks(env_names, get_passport, env_build, env_inventory_init, env_inventory_content): - for env in env_names.split("\n"): + environment_names = split_multi_value_param(env_names) + for env in environment_names: # now we are using only complex environment names that contain both cluster_name and environment_name if env.count('/') != 1: logger.fatal(f"Wrong env_name given: {env}. Env_name should contain both cloud name and environment name by pattern '/'") diff --git a/creds_rotation/build/requirements.txt b/creds_rotation/build/requirements.txt index 5c02cea82..e7aafecc4 100644 --- a/creds_rotation/build/requirements.txt +++ b/creds_rotation/build/requirements.txt @@ -1,6 +1,6 @@ pytest==7.4.3 PyYAML==6.0.1 -jsonschema==4.19.1 +jsonschema==4.24.1 attrs==23.2.0 referencing==0.33.0 rpds-py==0.17.1 diff --git a/dependencies/tests_requirements.txt b/dependencies/tests_requirements.txt index 73b663ee1..c39f45cf8 100644 --- a/dependencies/tests_requirements.txt +++ b/dependencies/tests_requirements.txt @@ -16,15 +16,13 @@ ruamel.yaml==0.18.5 ruamel.yaml.clib==0.2.8 ruyaml==0.91.0 jschon==0.11.0 -jsonschema==4.19.1 +jsonschema==4.24.1 diagrams==0.23.3 attrs==23.2.0 referencing==0.33.0 rpds-py==0.17.1 jsonschema-specifications==2023.12.1 cryptography==41.0.3 -ansible-core==2.17.12 -ansible_runner==2.3.5 pytest==7.4.3 junitparser==3.1.2 hiyapyco==0.6.0 diff --git a/devtools/tests/Dockerfile b/devtools/tests/Dockerfile index f9c98e30c..c71e3e1d9 100644 --- a/devtools/tests/Dockerfile +++ b/devtools/tests/Dockerfile @@ -16,7 +16,9 @@ RUN curl -LO https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9. COPY dependencies/pip.conf /etc/pip.conf COPY dependencies/sources.list /etc/apt/sources.list -# Copy dependencies and install them -COPY dependencies/tests_requirements.txt /tmp/ -RUN pip install --no-cache-dir "uv>=0.9.5" && uv pip install --system --no-cache-dir -r /tmp/tests_requirements.txt - +# PyPI test dependencies only (repo packages are installed editable at container start via up.sh) +COPY dependencies/tests_requirements.txt /tmp/tests_requirements.txt +# hadolint ignore=DL3013,DL3042 +RUN pip install --no-cache-dir "uv>=0.9.5" && \ + uv pip install --system --upgrade pip "setuptools<82" wheel && \ + uv pip install --system --no-cache-dir -r /tmp/tests_requirements.txt diff --git a/devtools/tests/run.sh b/devtools/tests/run.sh index 5a12db405..e936e4720 100755 --- a/devtools/tests/run.sh +++ b/devtools/tests/run.sh @@ -1,18 +1,30 @@ #!/bin/bash -set -euxo +set -euxo pipefail -cd "$CI_PROJECT_DIR" +cd "${CI_PROJECT_DIR}" -# Run tests -cd python/envgene/envgenehelper -pytest --capture=no -W ignore::DeprecationWarning --junitxml=../../../junit.xml -cd ../../.. -mv junit.xml junit_envgenehelper.xml +export PYTHONPATH=${CI_PROJECT_DIR} +export FULL_ENV_NAME="sdp-dev/env-1" +export BG_STATE="" -cd scripts/build_env -pytest --capture=no -W ignore::DeprecationWarning --junitxml=../../junit.xml -cd ../.. -mv junit.xml junit_build_env.xml +rm -f junit.xml junit_*.xml -# Merge results -python -m junitparser merge junit_build_env.xml junit_envgenehelper.xml junit.xml +run_pytest_suite() { + local name="$1" + local dir="$2" + ( + cd "${CI_PROJECT_DIR}/${dir}" + pytest --capture=no -W ignore::DeprecationWarning --junitxml="${CI_PROJECT_DIR}/junit.xml" + ) + mv "${CI_PROJECT_DIR}/junit.xml" "${CI_PROJECT_DIR}/junit_${name}.xml" +} + +run_pytest_suite envgenehelper python/envgene/envgenehelper +run_pytest_suite pipegene build_pipegene/scripts +run_pytest_suite artifact_searcher python/artifact-searcher/artifact_searcher +run_pytest_suite bg_manage scripts/bg_manage +run_pytest_suite build_env scripts/build_env +run_pytest_suite cred_rotation creds_rotation/scripts +run_pytest_suite sbom_retention build_effective_set_generator/scripts + +junitparser merge junit_*.xml junit.xml diff --git a/devtools/tests/up.sh b/devtools/tests/up.sh index f57ba0a22..d80946ada 100755 --- a/devtools/tests/up.sh +++ b/devtools/tests/up.sh @@ -1,3 +1,5 @@ #!/bin/bash +set -euo pipefail + chmod +x /workspace/python/build_modules.sh /workspace/python/build_modules.sh diff --git a/docs/README.md b/docs/README.md index 56a7494bd..1f7f1f294 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,8 +2,10 @@ - [EnvGene Documentation](#envgene-documentation) - [Getting Started](#getting-started) + - [Tutorials](#tutorials) - [Core Concepts](#core-concepts) - [How-To Guides](#how-to-guides) + - [Migrations](#migrations) - [Advanced Features](#advanced-features) - [Examples \& Samples](#examples--samples) - [Development](#development) @@ -12,6 +14,11 @@ - [**Quick Start Guide**](/README.md#getting-started) - Create your first Environment +## Tutorials + +- [**Understanding the Effective Set**](/docs/tutorials/effective-set.md) - Trace how parameters from Tenant, Cloud, Namespace, Application, and SBOM sources are merged into the final Effective Set; learn to read traceability comments and debug wrong values +- [**Managing Resource Profiles**](/docs/tutorials/resource-profiles.md) - End-to-end walkthrough: from Baseline to Template Override to Environment-Specific Override, including `template_override`, `overrides-parent`, and result verification + ## Core Concepts - [**EnvGene Objects**](/docs/envgene-objects.md) - What are EnvGene objects and how they work @@ -36,28 +43,40 @@ - [**Override Template Parameters**](/docs/how-to/environment-specific-parameters.md) - Override template parameters for specific environments - [**Configure Resource Profiles**](/docs/how-to/configure-resource-profiles.md) - Configure performance parameters for different environment types +**Effective Set:** + +- [**Generate an Effective Set**](/docs/how-to/generate-effective-set.md) - Trigger Effective Set generation from a Solution Descriptor artifact and template version + **Advanced Configuration:** - [**Configure Namespace Names for Sites**](/docs/how-to/configure-ns-names-for-sites.md) - Site-specific namespace naming +- [**Filter Namespaces in Template Descriptor**](/docs/how-to/filter-ns-in-template-descriptor.md) - Generate Environments with selected namespaces only - [**Credential Encryption**](/docs/how-to/credential-encryption.md) - Secure credential storage and rotation + +## Migrations + - [**Migrate to Dot-Notated Parameters**](/docs/how-to/dot-notated-parameter-migration.md) - Parameter format migration +- [**Migrate SBOM Storage to Per-Application Layout**](/docs/how-to/sbom-storage-migration.md) - Transition to per-application SBOM directory layout when upgrading EnvGene ## Advanced Features - [**Solution Descriptor Processing**](/docs/features/sd-processing.md) - Manage [Solution Descriptor](/docs/envgene-objects.md#solution-descriptor) for your Environments - [**Effective Set Calculation**](/docs/features/calculator-cli.md) - Calculate the [Effective Set](/docs/features/calculator-cli.md#effective-set-v20) +- [**Custom Params**](/docs/instance-pipeline-parameters.md#custom_params) for session-scoped overrides - [**Application and Registry Definition**](/docs/features/app-reg-defs.md) - Describe how applications and registries are defined and referenced - [**Environment Inventory Generation**](/docs/features/env-inventory-generation.md) - Auto-generate [Environment Inventory](/docs/envgene-configs.md#env_definitionyml) - [**Environment Instance Generation**](/docs/features/environment-instance-generation.md) - Generate Environment Instances from templates and inventories (including BG support) - [**Credential Rotation**](/docs/features/cred-rotation.md) - Automate [Credential](/docs/envgene-objects.md#credential) rotation - [**Namespace Render Filter**](/docs/features/namespace-render-filtering.md) - Render only selected [Namespaces](/docs/envgene-objects.md#namespace) +- [**Namespace Filtering in Template Descriptor**](/docs/features/namespace-filtering-in-template-descriptor.md) - Filter namespaces during Template Descriptor rendering - [**System Certificate Configuration**](/docs/features/system-certificate.md) - Auto-config system certs for internal registries or TLS services - [**Template Override**](/docs/features/template-override.md) - Use a base Environment template and override parts as needed - [**Automatic Environment Name Derivation**](/docs/features/auto-env-name-derivation.md) - Auto-detect Environment name from folder structure -- [**Template Inheritance**](/docs/features/template-inheritance.md) - Advanced Environment template patterns +- [**Template Composition**](/docs/features/template-composition.md) - Advanced Environment template patterns - [**Blue-Green Deployment**](/docs/features/blue-green-deployment.md) - BG domains, state management, and `bg_manage` pipeline job - [**Resource Profiles**](/docs/features/resource-profile.md) - Baselines and overrides for performance parameters - [**SBOM**](/docs/features/sbom.md) - CycloneDX-based artifact and parameter exchange for EnvGene +- [**SBOM Retention**](/docs/features/sbom-retention.md) - Automatic cleanup of cached SBOM files to manage repository size ## Examples & Samples diff --git a/docs/analysis/application-manifest-build-cli.md b/docs/analysis/application-manifest-build-cli.md index 7e98681c9..a5eb49bb9 100644 --- a/docs/analysis/application-manifest-build-cli.md +++ b/docs/analysis/application-manifest-build-cli.md @@ -86,7 +86,7 @@ flowchart TD ## Requirements 1. The CLI must generate AM that validates against [JSON Schema](/schemas/application-manifest.schema.json) -2. The CLI must use as input [Registry Definition v2.0](/schemas/regdef-v2.schema.json) +2. The CLI must use as input [Registry Definition v2.0](/python/envgene/envgenehelper/schemas/regdef-v2.schema.json) (bundled in envgenehelper package) 3. For each application entity listed below, an AM component with the corresponding MIME type must be generated: 1. "Service" -> `application/vnd.qubership.standalone-runnable` 2. Docker image -> `application/vnd.docker.image` @@ -498,7 +498,7 @@ Each individual registry is described by a separate `yaml` file in the `/configu The `name` attribute must match the filename without the extension. -[Registry Definition v2.0](/schemas/regdef-v2.schema.json) +[Registry Definition v2.0](/python/envgene/envgenehelper/schemas/regdef-v2.schema.json) (see envgene-objects.md for schema details) [Example](/examples/sandbox.yml) diff --git a/docs/analysis/documentation-gaps-analysis.md b/docs/analysis/documentation-gaps-analysis.md index 9068bf6d3..115eeef81 100644 --- a/docs/analysis/documentation-gaps-analysis.md +++ b/docs/analysis/documentation-gaps-analysis.md @@ -253,10 +253,10 @@ How-to guides are **task-oriented** recipes that guide users through steps to so - Common use cases - Best practices -- ❌ **How to use Template Inheritance** +- ❌ **How to use Template Composition** - Create parent template - Create child template - - Configure inheritance + - Configure composition - Override parent elements --- @@ -293,7 +293,7 @@ How-to guides are **task-oriented** recipes that guide users through steps to so ##### Advanced Scenarios (0% coverage) - ❌ **How to use Template Override** - - When to use override vs inheritance + - When to use override vs composition - Configure override - Override specific sections - Test overrides @@ -581,11 +581,11 @@ why things are the way they are. - Data flow between repositories - Why this architecture -- ❌ **How Template Inheritance works and when to use it** - - Inheritance concept +- ❌ **How Template Composition works and when to use it** + - Composition concept - Use cases - Design patterns - - When NOT to use inheritance + - When NOT to use composition - ❌ **EnvGene Data Model: From Template to Effective Set** - Complete data flow @@ -610,7 +610,7 @@ why things are the way they are. - Consumers of Effective Set - ❌ **ParameterSets concept and parameter hierarchy** - - Parameter inheritance model + - Parameter composition model - Merge strategies - Precedence rules - Design rationale @@ -682,7 +682,7 @@ why things are the way they are. - Complementary use - Choosing between them -- ❌ **Template Inheritance vs Template Override** +- ❌ **Template Composition vs Template Override** - When to use each - Pros and cons - Can they be combined? @@ -975,7 +975,7 @@ Troubleshooting guides reduce support load. │ ├── comparisons/ # 🆕 NEW section │ │ ├── envgene-vs-helm.md │ │ ├── envgene-vs-kustomize.md -│ │ └── inheritance-vs-override.md +│ │ └── composition-vs-override.md │ └── advanced-topics/ # 🆕 NEW section │ ├── scaling-to-100-environments.md │ ├── security-best-practices.md @@ -1243,7 +1243,7 @@ Create documentation for each primary user journey: 3. 📕 `reference/templates/jinja-filters.md` - REFERENCE 4. 📕 `reference/objects/template-descriptor.md` - REFERENCE (exists as part of envgene-objects.md) 5. 📙 `explanation/patterns/template-repository-organization.md` - EXPLANATION -6. 📗 `how-to/advanced/template-inheritance.md` - HOW-TO +6. 📗 `how-to/advanced/template-composition.md` - HOW-TO **Current gaps:** Steps 1, 3, 5, 6 missing diff --git a/docs/dev/app_reg_def_job.drawio.png b/docs/dev/app_reg_def_job.drawio.png new file mode 100644 index 000000000..3ee00509f Binary files /dev/null and b/docs/dev/app_reg_def_job.drawio.png differ diff --git a/docs/dev/global-pre-commit-hooks.md b/docs/dev/global-pre-commit-hooks.md new file mode 100644 index 000000000..809ee3be4 --- /dev/null +++ b/docs/dev/global-pre-commit-hooks.md @@ -0,0 +1,108 @@ +# Global pre-commit hooks + +- [Global pre-commit hooks](#global-pre-commit-hooks) + - [Description](#description) + - [Prerequisites](#prerequisites) + - [Step 1: Clone pre-commit-global](#step-1-clone-pre-commit-global) + - [Step 2: Point Git at the global hooks directory](#step-2-point-git-at-the-global-hooks-directory) + - [Step 3: grand-report.json](#step-3-grand-reportjson) + - [What runs on commit](#what-runs-on-commit) + - [Disable global hooks](#disable-global-hooks) + - [References](#references) + +## Description + +This guide shows how to register [pre-commit-global](https://github.com/exadmin/pre-commit-global) hooks globally on your machine so every Git repository can use shared hook logic before your normal pre-commit runs. Hook scripts live in this repository under `hooks-global/`. Full install details and upstream updates are documented in the [pre-commit-global readme](https://github.com/exadmin/pre-commit-global). + +## Prerequisites + +| Requirement | Purpose | +|-----------------------------|------------------------------------------------------------------------------------------------------------| +| Git | Required. See [git-scm.com](https://git-scm.com/install/). | +| Java (JDK or JRE) | Required by the hook toolchain (upstream tests with a recent JDK). | +| `CYBER_FERRET_PASSWORD` | Only if CyberFerret runs; see the note below. | + +> [!NOTE] +> For the `CYBER_FERRET_PASSWORD` value or questions about it, contact **Andrei Rudchenko**. + +## Step 1: Clone pre-commit-global + +Choose a directory you intend to keep (for example `~/tools/global-git-hooks` on Linux or macOS, or `C:\Tools\global-git-hooks` on Windows). Later, if you move or delete this folder you must repeat [Step 2](#step-2-point-git-at-the-global-hooks-directory). + +**Clone into the directory root:** + +```bash +mkdir -p ~/tools/global-git-hooks +cd ~/tools/global-git-hooks +git clone https://github.com/exadmin/pre-commit-global . +``` + +**Or clone into a named subfolder:** + +```bash +mkdir -p ~/tools +cd ~/tools +git clone https://github.com/exadmin/pre-commit-global my-global-hooks +cd my-global-hooks +``` + +On Windows Command Prompt, `mkdir`, `cd` to your chosen folder, run the same `git clone`, then `cd` into the clone. + +Stay in this clone directory when running the commands in the next step. + +## Step 2: Point Git at the global hooks directory + +Configure `core.hooksPath` to the `hooks-global` directory inside your clone. + +**Linux and macOS:** + +```bash +git config --global core.hooksPath "$(pwd)/hooks-global" +git config --global core.hooksPath +``` + +**Windows (cmd):** + +```bat +git config --global core.hooksPath "%cd%\hooks-global" +git config --global core.hooksPath +``` + +The second command prints the value Git stored so you can confirm the path. + +> [!TIP] +> Alternatively, run `linux_register_this_folder_as_global_hooks.sh` (Linux or macOS) or `win_register_*.cmd` (Windows) from your clone root so `core.hooksPath` points at this clone's `hooks-global` folder, instead of typing the `git config` commands above. + +## Step 3: grand-report.json + +The **`.qubership/grand-report.json`** file at the repository root is **added by Andrei Rudchenko**. It is required on the CyberFerret-related hook path and holds ignores and exclusions for signatures as needed. + +Use `CYBER_FERRET_PASSWORD` as in [Prerequisites](#prerequisites). + +> [!NOTE] +> For updates to this file or questions about it, contact **Andrei Rudchenko**. + +## What runs on commit + +When you run `git commit -m "your message"`: + +1. Global hooks run (including an online hook-update check). +2. If `.pre-commit-config.yaml` exists, **pre-commit** runs with that config. +3. If pre-commit passes or there is no config, the repository's **`.git/hooks/pre-commit`** runs if present. + +If any check fails, the commit stops until you fix the issue or adjust configuration and exclusions. + +## Disable global hooks + +To stop using global hooks machine-wide: + +```bash +git config --global --unset core.hooksPath +``` + +## References + +| Resource | Link | +|-----------------------------------------|-----------------------------------------------------------------------------------------------------------------------| +| pre-commit-global overview and readme | [github.com/exadmin/pre-commit-global](https://github.com/exadmin/pre-commit-global) | +| pre-commit framework | [pre-commit.com](https://pre-commit.com/) | diff --git a/docs/dev/job-artifacts.md b/docs/dev/job-artifacts.md new file mode 100644 index 000000000..95bd89aa3 --- /dev/null +++ b/docs/dev/job-artifacts.md @@ -0,0 +1,51 @@ +# Job Artifacts + +## Overview + +GitLab pipeline jobs pass state between each other through artifacts without repeated Git checkouts. This document explains the mechanism and requirements. + +Artifact size limit: **1500 MB** + +## Git Checkout Strategy + +### First Job in Pipeline + +1. Performs `git checkout` +2. Gets fresh copy of repository +3. Modifies files (optional) +4. Saves required paths to job artifacts + +### Intermediate Jobs + +- Do NOT checkout repository +- Receive files from previous job's artifacts +- Modify files (optional) +- Save required paths to job artifacts + +### git_commit_job + +- Receives files from previous job's artifacts +- Does `git init` and `git pull` (retrieves current repository state from remote) +- Copies files from job's artifacts (overwrites pulled files with changes) +- Commits and pushes changes + +## Required Artifact Paths + +All jobs in the pipeline save **only** these paths to artifacts: + +- `/environments/` + +- `/configuration/` +- `/sboms/` +- `/templates/` + +These paths are: + +1. Modified by various jobs during pipeline execution +2. Needed by downstream jobs +3. Committed to Git by `git_commit_job` diff --git a/docs/dev/unify-logging.md b/docs/dev/unify-logging.md index a450df88f..f4c49dcac 100644 --- a/docs/dev/unify-logging.md +++ b/docs/dev/unify-logging.md @@ -43,7 +43,7 @@ All other modules import and use it. A new parameter was added to control logging behavior. -[Link to documentation](https://github.com/Netcracker/qubership-envgene/blob/a823f450a671d058813991b218b9afde59f6db41/docs/envgene-repository-variables.md#envgene_log_level) +[Link to documentation](/docs/envgene-repository-variables.md#envgene_log_level) --- @@ -67,4 +67,4 @@ A script was added that: - Runs at the start of every generated job - Logs input parameters -[How it was implemented](https://github.com/Netcracker/qubership-envgene/blob/main/build_pipegene/scripts/pipeline_helper.py#L47-L50) +[How it was implemented](/build_pipegene/scripts/pipeline_helper.py#L47-L50) diff --git a/docs/envgene-configs.md b/docs/envgene-configs.md index a99032d26..f87d0af03 100644 --- a/docs/envgene-configs.md +++ b/docs/envgene-configs.md @@ -225,6 +225,17 @@ artifact_definitions_discovery_mode: enum [`auto`, `true`, `false`] # `cmdb` - Application and Registry Definitions are discovered from a CMDB system (discovery procedure is not part of EnvGene Core). Discovery result is saved in repository # `auto` - Definitions are first searched in repository, if not found - discovered from CMDB. Discovery result is saved in repository app_reg_def_mode: enum [`auto`, `cmdb`, `local`] +# Optional +# SBOM retention configuration +# Triggers during Effective Set generation when repository reaches 1200 MB size threshold +sbom_retention: + # Optional. Default value - `false` + # Enable/disable SBOM retention cleanup + enabled: boolean + # Optional. Default value - `10` + # Number of latest versions to keep per application + # Used only when enabled is true + keep_versions_per_app: integer ``` ## `integration.yml` @@ -249,12 +260,12 @@ cp_discovery: # Mandatory # Authentication token for the discovery repository # Recommended to set via cred macro: - # envgen.creds.get().secret + # ${creds.get('').secret} token: string # Authentication token for EnvGene to access the instance repository # Required for EnvGene to commit changes to the instance repository # Recommended to set via cred macro: -# envgen.creds.get().secret +# ${creds.get('').secret} self_token: string ``` diff --git a/docs/envgene-objects.md b/docs/envgene-objects.md index e49160dc9..778a40f22 100644 --- a/docs/envgene-objects.md +++ b/docs/envgene-objects.md @@ -65,14 +65,21 @@ This object is a describes the structure of a solution, links to solution's comp The name of this file serves as the name of the Environment Template. In the Environment Inventory, this name is used to specify which Environment Template from the artifact should be used. -**Location:** Any YAML file located in the `/templates/env_templates/` folder is considered a Template Descriptor. +**Location:** Any YAML or Jinja file located in the `/templates/env_templates/` folder is considered a Template Descriptor. + +**Supported file extensions:** + +- `.yml` / `.yaml` — Static Template Descriptor +- `.yml.j2` / `.yaml.j2` — Jinja Template Descriptor (rendered before Environment Instance generation) + +When multiple Template Descriptors with the same base name but different extensions exist, EnvGene selects them in descending priority order: `yml.j2` > `yaml.j2` > `yml` > `yaml`. Jinja Template Descriptors enable conditional namespace inclusion. See [Namespace Filtering in Template Descriptor](/docs/features/namespace-filtering-in-template-descriptor.md) for details. It has the following structure: ```yaml # Optional -# Template Inheritance configuration -# See details in https://github.com/Netcracker/qubership-envgene/blob/main/docs/features/template-inheritance.md +# Template Composition configuration +# See details in https://github.com/Netcracker/qubership-envgene/blob/main/docs/features/template-composition.md parent-templates: # Optional # Value must be in `application:version` notation @@ -82,8 +89,8 @@ parent-templates: tenant: string # or tenant: - # Template Inheritance configuration - # See details in https://github.com/Netcracker/qubership-envgene/blob/main/docs/features/template-inheritance.md + # Template Composition configuration + # See details in https://github.com/Netcracker/qubership-envgene/blob/main/docs/features/template-composition.md parent: string # Mandatory # Can be specified either as direct template path (string) or as an object @@ -95,17 +102,19 @@ cloud: # Optional # Template Override configuration # See details in https://github.com/Netcracker/qubership-envgene/blob/main/docs/template-override.md - template_override: template_override: # Optional - # Template Inheritance configuration - # See details in https://github.com/Netcracker/qubership-envgene/blob/main/docs/features/template-inheritance.md + # Template Composition configuration + # See details in https://github.com/Netcracker/qubership-envgene/blob/main/docs/features/template-composition.md parent: string # Optional - # Template Inheritance configuration - # See details in https://github.com/Netcracker/qubership-envgene/blob/main/docs/features/template-inheritance.md + # Template Composition configuration + # See details in https://github.com/Netcracker/qubership-envgene/blob/main/docs/features/template-composition.md overrides-parent: + # Optional + # Override the name of the cloud in rendering result + name: string profile: override-profile-name: parent-profile-name: @@ -137,16 +146,19 @@ namespaces: # Optional # Name of Namespace in Parent Template - # See details in https://github.com/Netcracker/qubership-envgene/blob/main/docs/features/template-inheritance.md + # See details in https://github.com/Netcracker/qubership-envgene/blob/main/docs/features/template-composition.md name: string # Optional # Parent template name - # See details in https://github.com/Netcracker/qubership-envgene/blob/main/docs/features/template-inheritance.md + # See details in https://github.com/Netcracker/qubership-envgene/blob/main/docs/features/template-composition.md parent: string # Optional - # Template Inheritance configuration - # See details in https://github.com/Netcracker/qubership-envgene/blob/main/docs/features/template-inheritance.md + # Template Composition configuration + # See details in https://github.com/Netcracker/qubership-envgene/blob/main/docs/features/template-composition.md overrides-parent: + # Optional + # Override the name of the namespace in rendering result + name: string profile: override-profile-name: string parent-profile-name: string @@ -165,11 +177,73 @@ namespaces: #### Tenant Template -TBD +This is a Jinja template file used to render the [Tenant](#tenant) object. It defines tenant-level parameters for Environment Instance generation. + +The Tenant template must be developed so that after Jinja rendering, the result is a valid Tenant object according to the [schema](/schemas/tenant.schema.json). + +[Macros](/docs/template-macros.md) are available for use when developing the template. + +**Location:** The Tenant template is located at `/templates/env_templates/*/` + +**Example:** + +```yaml +name: "Applications" +registryName: "" +description: "For development" +owners: "{{ current_env.owners }}" +credential: "" +labels: [] +``` #### Cloud Template -TBD +This is a Jinja template file used to render the [Cloud](#cloud) object. It defines cluster-level parameters for Environment Instance generation. + +The Cloud template must be developed so that after Jinja rendering, the result is a valid Cloud object according to the [schema](/schemas/cloud.schema.json). + +[Macros](/docs/template-macros.md) are available for use when developing the template. + +**Location:** The Cloud template is located at `/templates/env_templates/*/` + +**Example:** + +```yaml +name: "{{ current_env.cloudNameWithCluster }}" +apiUrl: "{{ current_env.cluster.cloud_api_url }}" +apiPort: "{{ current_env.cluster.cloud_api_port }}" +privateUrl: "" +publicUrl: "{{ current_env.cluster.cloud_public_url }}" +dashboardUrl: "https://dashboard.{{ current_env.cluster.cloud_public_url }}" +labels: [] +defaultCredentialsId: "token" +protocol: "{{ current_env.cluster.cloud_api_protocol }}" +deployParameters: {} +e2eParameters: {} +technicalConfigurationParameters: {} +deployParameterSets: [] +e2eParameterSets: [] +technicalConfigurationParameterSets: [] +maasConfig: + credentialsId: "maas" + maasUrl: "http://maas-service-maas.{{ current_env.cluster.cloud_public_url }}" + maasInternalAddress: "http://maas-service.maas:8080" + enable: true +vaultConfig: + url: "" + credentialsId: "" + enable: false +dbaasConfigs: + - credentialsId: "dbaas" + apiUrl: 'http://dbaas-aggregator.dbaas:8080' + aggregatorUrl: 'https://aggregator-dbaas.{{ current_env.cluster.cloud_public_url }}' + enable: true +consulConfig: + tokenSecret: "consul-token" + publicUrl: 'https://consul.{{ current_env.cluster.cloud_public_url }}' + enabled: true + internalUrl: 'http://consul-server.consul:8500' +``` #### Namespace Template @@ -536,11 +610,319 @@ EnvGene validates each Environment Instance object against the corresponding [JS #### Tenant -TBD +The Tenant object holds tenant-level parameters describing the tenancy, including registry configuration, ownership information, and pipeline parameters. These parameters are common to all environments within the tenant. + +The Tenant object is used to generate Effective Set. + +The Tenant object is generated during Environment Instance generation based on: + +- [Tenant Template](#tenant-template) +- [Template ParamSet](#template-parameterset) +- [Instance ParamSet](#environment-specific-parameterset) + +For each parameter in the Tenant, a comment is added indicating the source Parameter Set from which this parameter originated. This is used for traceability in the generation of the environment instance. + +**Location:** `/environments///tenant.yml`. + +```yaml +# Mandatory +# Field is used to uniquely identify the Tenant +# The name of the tenant +name: string +# Mandatory +# Deprecated +# Not processed by EnvGene +registryName: string +# Optional +# Description of the tenant +# Used for documentation and identification purposes +description: string +# Optional +# Tenant owners +# Used to identify responsible parties for the tenant +owners: string +# Optional +# Deprecated +# Not processed by EnvGene +gitRepository: string +# Optional +# Deprecated +# Not processed by EnvGene +defaultBranch: string +# Optional +# The identifier for credentials used by the deployment +# Used for authentication when performing deployment operations +credential: string +# Optional +# List of labels for Tenant +# A list of labels that should be applied to the tenant +# Used for filtering, organization, and grouping +labels: list +# Optional +# Deprecated +# Not processed by EnvGene +globalE2EParameters: + # Optional + # Deprecated + # Not processed by EnvGene + pipelineDefaultRecipients: string + # Optional + # Deprecated + # Not processed by EnvGene + recipientsStrategy: string + # Optional + # Deprecated + # Not processed by EnvGene + mergeTenantsAndE2EParameters: boolean + # Optional + # Deprecated + # Not processed by EnvGene + environmentParameters: hashmap +# Optional +# Deprecated +# Not processed by EnvGene +deployParameters: hashmap +``` + +**Example:** + +```yaml +# The contents of this file is generated from template artifact: sample-template:v1.2.3. +# Contents will be overwritten by next generation. +# Please modify this contents only for development purposes or as workaround. +name: "tenant" +registryName: "" +description: "Composite Full Sample" +owners: "Qubership team" +credential: "" +labels: [] +``` + +[Tenant JSON schema](/schemas/tenant.schema.json) #### Cloud -TBD +The Cloud object holds cluster-level parameters describing the cluster and platform applications installed in it. These parameters are common to all namespaces in the environment. + +The Cloud object is used to generate Effective Set. + +The Cloud object is generated during Environment Instance generation based on: + +- [Cloud Template](#cloud-template) +- [Template ParamSet](#template-parameterset) +- [Instance ParamSet](#environment-specific-parameterset) +- [Cloud Passport](/docs/envgene-objects.md#cloud-passport) data (when used) + +For each parameter in the Cloud, a comment is added indicating the source Parameter Set from which this parameter originated. This is used for traceability in the generation of the environment instance. + +**Location:** `/environments///cloud.yml`. + +```yaml +# Mandatory +# The name of the cloud configuration +# Typically combines cluster and environment name +name: string +# Mandatory +# The URL of the API endpoint of the cloud +# Used to connect to the Kubernetes cluster API server +apiUrl: string +# Mandatory +# The port on which the API runs +# Used to connect to the Kubernetes cluster API server +apiPort: integer|string +# Optional +# The private-facing URL for internal access +# Used to form service URLs accessible from within the cluster +privateUrl: string +# Optional +# The public-facing URL for external access +# Used to form service URLs accessible from outside the cluster +# Calculator macros are generated based on this URL +publicUrl: string +# Mandatory +# The URL for accessing the cloud's k8s dashboard +# Used for monitoring and management +dashboardUrl: string +# Mandatory +# A list of labels for categorizing or tagging the cloud +# Used for filtering, organization, and grouping +labels: list +# Mandatory +# The identifier for credentials used by the deployment +# Used for authentication when performing deployment +defaultCredentialsId: string +# Mandatory +# The communication protocol used +# HTTP or HTTPS +protocol: string +# Optional +# Deprecated +# Not processed by EnvGene +version: number +# Optional +# Deprecated +# Not processed by EnvGene +dbMode: string +# Optional +# Deprecated +# Not processed by EnvGene +databases: array +# Optional +# Deprecated +# Not processed by EnvGene +mergeDeployParametersAndE2EParameters: boolean +# Mandatory +# Configuration for the monitoring-as-a-service (MaaS) +maasConfig: + # Optional + # Credentials identifier for MaaS + # Used for authentication when accessing MaaS + credentialsId: string + # Mandatory + # Flag to enable or disable MaaS + # Controls whether MaaS-related parameters appear in the Effective Set + enable: boolean + # Optional + # URL for accessing MaaS + # Used to configure external access to MaaS + maasUrl: string + # Optional + # Internal address for MaaS + # Used to configure internal cluster access to MaaS + maasInternalAddress: string +# Mandatory +# Configuration for the vault service +vaultConfig: + # Optional + # Credentials identifier for the vault + # Used for authentication when accessing Vault + credentialsId: string + # Mandatory + # Flag to enable or disable vault integration + # Controls whether Vault-related parameters appear in the Effective Set + enable: boolean + # Optional + # The vault service URL + # Used to configure access to Vault + url: string +# Optional +# Database-as-a-service (DBaaS) configurations +# Multiple DBaaS instances can be configured +dbaasConfigs: + - # Optional + # Credentials identifier for DBaaS + # Used for authentication when accessing DBaaS + credentialsId: string + # Mandatory + # Flag to enable or disable DBaaS + # Controls whether DBaaS-related parameters appear in the Effective Set + enable: boolean + # Optional + # API URL for DBaaS + # Used to configure internal cluster access to DBaaS + apiUrl: string + # Optional + # URL for the DBaaS aggregator + # Used to configure external access to DBaaS + aggregatorUrl: string +# Mandatory +# Configuration for Consul service integration +consulConfig: + # Optional + # Secret token for Consul authentication + # Used for authentication when accessing Consul + tokenSecret: string + # Mandatory + # Flag to enable or disable Consul integration + # Controls whether Consul-related parameters appear in the Effective Set + enabled: boolean + # Optional + # The public URL for accessing Consul + # Used to configure external access to Consul + publicUrl: string + # Optional + # The internal URL for accessing Consul + # Used to configure internal cluster access to Consul + internalUrl: string +# Optional +# Key-value pairs of deployment parameters at the cloud level +# Used to set parameters that will be used for rendering Helm charts of applications in this cloud +deployParameters: hashmap +# Optional +# Key-value pairs of e2e parameters at the cloud level +# Used to configure the systems/pipelines managing the Environment lifecycle for this cloud +e2eParameters: hashmap +# Optional +# Key-value pairs of technical configuration parameters at the cloud level +# Used to set parameters that can be applied to the application at runtime +# without redeployment for this cloud +technicalConfigurationParameters: hashmap +# Optional +# List of deployment Parameter Set names to include at the cloud level +# Used to set parameters that will be used for rendering Helm charts of applications in this cloud +deployParameterSets: list +# Optional +# List of e2e Parameter Set names to include at the cloud level +# Used to configure the systems/pipelines managing the Environment lifecycle for this cloud +e2eParameterSets: list +# Optional +# List of technical configuration Parameter Set names to include at the cloud level +# Used to include predefined sets of parameters that can be applied to the application at runtime +# without redeployment for this cloud +technicalConfigurationParameterSets: list +``` + +**Example:** + +```yaml +# The contents of this file is generated from template artifact: sample-template:v1.2.3. +# Contents will be overwritten by next generation. +# Please modify this contents only for development purposes or as workaround. +name: "cluster_01_env_01" +apiUrl: "api.cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 +apiPort: "6443" # cloud passport: cluster-01 version: 1.5 +privateUrl: "cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 +publicUrl: "cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 +dashboardUrl: "https://dashboard.cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 +labels: [] +defaultCredentialsId: "cloud-deploy-sa-token" # cloud passport: cluster-01 version: 1.5 +protocol: "https" # cloud passport: cluster-01 version: 1.5 +maasConfig: + credentialsId: "maas-cred" # cloud passport: cluster-01 version: 1.5 + enable: true # cloud passport: cluster-01 version: 1.5 + maasUrl: "http://maas.cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 + maasInternalAddress: "http://maas.maas:8080" # cloud passport: cluster-01 version: 1.5 +vaultConfig: + credentialsId: "" + enable: false + url: "" +dbaasConfigs: + - credentialsId: "dbaas-cred" # cloud passport: cluster-01 version: 1.5 + enable: true # cloud passport: cluster-01 version: 1.5 + apiUrl: "http://dbaas.dbaas:8080" # cloud passport: cluster-01 version: 1.5 + aggregatorUrl: "https://dbaas.cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 +consulConfig: + tokenSecret: "consul-cred" # cloud passport: cluster-01 version: 1.5 + enabled: true # cloud passport: cluster-01 version: 1.5 + publicUrl: "http://consul.consul:8080" # cloud passport: cluster-01 version: 1.5 + internalUrl: "http://consul.consul:8080" # cloud passport: cluster-01 version: 1.5 +deployParameters: + CLOUD_DASHBOARD_URL: "https://dashboard.cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 + CMDB_URL: "https://cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 + CONSUL_ENABLED: "true" # cloud passport: cluster-01 version: 1.5 + MAVEN_REPO_URL: "https://artifactory.qubership.org" # cloud passport: cluster-01 version: 1.5 + MONITORING_ENABLED: "true" # cloud passport: cluster-01 version: 1.5 + STORAGE_RWO_CLASS: "standard" # cloud passport: cluster-01 version: 1.5 + ZOOKEEPER_ADDRESS: "zookeeper.zookeeper:2181" # cloud passport: cluster-01 version: 1.5 +e2eParameters: + CLOUD_LEVEL_PARAM_1: "cloud-level-value-1" # paramset: cloud-level-params version: 25.1 source: instance +technicalConfigurationParameters: {} +deployParameterSets: [] +e2eParameterSets: [] +technicalConfigurationParameterSets: [] +``` + +[Cloud JSON schema](/schemas/cloud.schema.json) #### Namespace @@ -1490,7 +1872,8 @@ registry: # Supports advanced authentication methods including public cloud registries authConfig: : - # Mandatory + # Optional + # Not used in case of `authMethod: anonymous` # Pointer to the EnvGene Credential object. # Depending on `authType`, it can be: # access key (username) + secret (password) for longLived @@ -1500,17 +1883,18 @@ registry: # Public cloud registry authentication strategy # Used in case of public cloud registries authType: enum [ shortLived, longLived ] - # Optional - # Public cloud registry type - # Used in case of public cloud registries - provider: enum [ aws, azure, gcp ] - # Optional + # Mandatory + # Registry type + provider: enum [ aws, azure, gcp, nexus, artifactory ] + # Mandatory # In case of non-cloud public registries, `user_pass` is used # In case of public cloud registries valid values, depends on `provider`: - # `aws`: `secret` or `assume_role` - # `gcp`: `federation` or `service_account` - # `azure`: `oauth2` - authMethod: enum [ secret, assume_role, federation, service_account, oauth2, user_pass ] + # `nexus`: `user_pass` or `anonymous` + # `artifactory`: `user_pass` or `anonymous` + # `aws`: `secret`, `assume_role` or `anonymous` + # `gcp`: `federation`, `service_account` or `anonymous` + # `azure`: `oauth2` or `anonymous` + authMethod: enum [ secret, assume_role, federation, service_account, oauth2, user_pass, anonymous ] # Optional # Region of the AWS cloud # Used with `provider: aws` only @@ -1575,28 +1959,31 @@ registry: azureArtifactsResource: string # Mandatory mavenConfig: - # Optional + # Mandatory # Pointer to authentication config described in `authConfig` section - # Cannot be set if anonymous access is used authConfig: string # Mandatory # Domain name of the registry repositoryDomainName: string - # Mandatory + # Optional + # Used in case of provider nexus or artifactory only # Snapshot repository name # EnvGene checks repositories in this order: release -> staging -> snapshot # It stops when it finds the artifact targetSnapshot: string - # Mandatory + # Optional + # Used in case of provider nexus or artifactory only # Staging repository name targetStaging: string - # Mandatory + # Optional + # Used in case of provider nexus or artifactory only # Release repository name targetRelease: string - # Mandatory + # Optional + # Used in case of provider nexus or artifactory only # Snapshot Maven repository group name snapshotGroup: string - # Mandatory + # Optional # Release Maven repository group name releaseGroup: string ``` @@ -1613,6 +2000,7 @@ registry: authConfig: maven-auth: authType: longLived + provider: nexus authMethod: user_pass credentialsId: "artifactory-cred" mavenConfig: @@ -1717,6 +2105,42 @@ registry: releaseGroup: "maven-releases-group" ``` +**Authentication Configuration Dependencies:** + +The `authConfig` section has complex dependencies between attributes. The following table shows which fields are required based on `provider` and `authMethod` values: + +| Field | Condition | Required | +|--------------------------|-----------------------------------------------------|---------------| +| `provider` | Always | **REQUIRED** | +| `authMethod` | Always | **REQUIRED** | +| `credentialsId` | `authMethod != "anonymous"` | **REQUIRED** | +| `authType` | `provider IN ["aws", "azure", "gcp"]` | OPTIONAL | +| `awsRegion` | `provider == "aws"` | OPTIONAL | +| `awsDomain` | `provider == "aws"` (required for CodeArtifact) | **REQUIRED** | +| `awsRoleARN` | `provider == "aws" AND authMethod == "assume_role"` | **REQUIRED** | +| `awsRoleSessionPrefix` | `provider == "aws" AND authMethod == "assume_role"` | OPTIONAL | +| `gcpOIDC` | `provider == "gcp" AND authMethod == "federation"` | **REQUIRED** | +| `gcpOIDC.URL` | Inside `gcpOIDC` | **REQUIRED** | +| `gcpOIDC.customParams` | Inside `gcpOIDC` | OPTIONAL | +| `gcpRegProject` | `provider == "gcp" AND authMethod == "federation"` | OPTIONAL | +| `gcpRegPoolId` | `provider == "gcp" AND authMethod == "federation"` | OPTIONAL | +| `gcpRegProviderId` | `provider == "gcp" AND authMethod == "federation"` | OPTIONAL | +| `gcpRegSAEmail` | `provider == "gcp" AND authMethod == "federation"` | OPTIONAL | +| `azureTenantId` | `provider == "azure"` | OPTIONAL | +| `azureACRResource` | `provider == "azure"` | OPTIONAL | +| `azureACRName` | `provider == "azure"` (required for ACR) | **REQUIRED** | +| `azureArtifactsResource` | `provider == "azure"` | OPTIONAL | + +**Valid `authMethod` values per `provider`:** + +| Provider | Valid authMethod values | +|---------------|----------------------------------------------| +| `nexus` | `user_pass`, `anonymous` | +| `artifactory` | `user_pass`, `anonymous` | +| `aws` | `secret`, `assume_role`, `anonymous` | +| `gcp` | `federation`, `service_account`, `anonymous` | +| `azure` | `oauth2`, `anonymous` | + [Artifact Definition v2.0 JSON schema](/schemas/artifact-definition-v2.schema.json) ### Registry Definition @@ -1888,7 +2312,8 @@ name: string # Authentication configs authConfig: : - # Mandatory + # Optional + # Not used in case of `authMethod: anonymous` # Pointer to the EnvGene Credential object. # Depending on `authType`, it can be: # access key (username) + secret (password) for longLived @@ -1898,17 +2323,18 @@ authConfig: # Public cloud registry authentication strategy # Used in case of public cloud registries authType: enum [ shortLived, longLived ] - # Optional - # Public cloud registry type - # Used in case of public cloud registries - provider: enum [ aws, azure, gcp ] - # Optional + # Mandatory + # Registry type + provider: enum [ aws, azure, gcp, nexus, artifactory ] + # Mandatory # In case of non-cloud public registries, `user_pass` is used # In case of public cloud registries valid values, depends on `provider`: - # `aws`: `secret` or `assume_role` - # `gcp`: `federation` or `service_account` - # `azure`: `oauth2` - authMethod: enum [ secret, assume_role, federation, service_account, oauth2, user_pass ] + # `nexus`: `user_pass` or `anonymous` + # `artifactory`: `user_pass` or `anonymous` + # `aws`: `secret`, `assume_role` or `anonymous` + # `gcp`: `federation`, `service_account` or `anonymous` + # `azure`: `oauth2` or `anonymous` + authMethod: enum [ secret, assume_role, federation, service_account, oauth2, user_pass, anonymous ] # Optional # Region of the AWS cloud # Used with `provider: aws` only @@ -1959,6 +2385,10 @@ authConfig: # Used with `provider: azure` only azureTenantId: string # Optional + # Region of the GCP cloud + # Used with `provider: gcp` only + gcpRegion: string + # Optional # Target resource for ACR # Used with `provider: azure` only azureACRResource: string @@ -1973,33 +2403,35 @@ authConfig: azureArtifactsResource: string # Mandatory mavenConfig: - # Optional + # Mandatory # Pointer to authentication config described in `authConfig` section - # Cannot be set in if anonymous access is used authConfig: string # Mandatory # Domain name of the registry repositoryDomainName: string - # Mandatory + # Optional + # Used in case of authMethod nexus or artifactory only # Snapshot Maven repository name targetSnapshot: string - # Mandatory + # Optional + # Used in case of authMethod nexus or artifactory only # Staging Maven repository name targetStaging: string - # Mandatory + # Optional + # Used in case of authMethod nexus or artifactory only # Release Maven repository name targetRelease: string - # Mandatory + # Optional + # Used in case of authMethod nexus or artifactory only # Snapshot Maven repository name snapshotGroup: string - # Mandatory + # Optional # Release Maven repository name releaseGroup: string -# Mandatory +# Optional dockerConfig: - # Optional + # Mandatory # Pointer to authentication config described in `authConfig` section - # Cannot be set in if anonymous access is used authConfig: string # Mandatory # URI for Docker snapshot registry @@ -2027,9 +2459,8 @@ dockerConfig: groupName: string # Optional helmConfig: - # Optional + # Mandatory # Pointer to authentication config described in `authConfig` section - # Cannot be set in if anonymous access is used authConfig: string # Mandatory # Domain name of the registry @@ -2042,9 +2473,8 @@ helmConfig: helmTargetRelease: string # Optional helmAppConfig: - # Optional + # Mandatory # Pointer to authentication config described in `authConfig` section - # Cannot be set in if anonymous access is used authConfig: string # Mandatory # Domain name of the registry @@ -2063,16 +2493,14 @@ helmAppConfig: helmDevRepoName: string # Optional goConfig: - # Optional + # Mandatory # Pointer to authentication config described in `authConfig` section - # Cannot be set in if anonymous access is used authConfig: string # Mandatory # Domain name of the registry repositoryDomainName: string # Mandatory - # Pointer to authentication config described in `authConfig` section - # Cannot be set in if anonymous access is used + # Go snapshot repository name goTargetSnapshot: string # Mandatory # Go release repository name @@ -2082,9 +2510,8 @@ goConfig: goProxyRepository: string # Optional npmConfig: - # Optional + # Mandatory # Pointer to authentication config described in `authConfig` section - # Cannot be set in if anonymous access is used authConfig: string # Mandatory # Domain name of the registry @@ -2097,16 +2524,14 @@ npmConfig: npmTargetRelease: string # Optional rawConfig: - # Optional + # Mandatory # Pointer to authentication config described in `authConfig` section - # Cannot be set in if anonymous access is used authConfig: string # Mandatory # Domain name of the registry repositoryDomainName: string # Mandatory - # Pointer to authentication config described in `authConfig` section - # Cannot be set in if anonymous access is used + # Raw snapshot repository name rawTargetSnapshot: string # Mandatory # Raw release repository name @@ -2119,6 +2544,43 @@ rawConfig: rawTargetProxy: string ``` +**Authentication Configuration Dependencies:** + +The `authConfig` section has complex dependencies between attributes. The following table shows which fields are required based on `provider` and `authMethod` values: + +| Field | Condition | Required | +|--------------------------|-----------------------------------------------------|---------------| +| `provider` | Always | **REQUIRED** | +| `authMethod` | Always | **REQUIRED** | +| `credentialsId` | `authMethod != "anonymous"` | **REQUIRED** | +| `authType` | `provider IN ["aws", "azure", "gcp"]` | OPTIONAL | +| `awsRegion` | `provider == "aws"` | OPTIONAL | +| `awsDomain` | `provider == "aws"` (required for CodeArtifact) | **REQUIRED** | +| `awsRoleARN` | `provider == "aws" AND authMethod == "assume_role"` | **REQUIRED** | +| `awsRoleSessionPrefix` | `provider == "aws" AND authMethod == "assume_role"` | OPTIONAL | +| `gcpOIDC` | `provider == "gcp" AND authMethod == "federation"` | **REQUIRED** | +| `gcpOIDC.URL` | Inside `gcpOIDC` | **REQUIRED** | +| `gcpOIDC.customParams` | Inside `gcpOIDC` | OPTIONAL | +| `gcpRegProject` | `provider == "gcp" AND authMethod == "federation"` | OPTIONAL | +| `gcpRegPoolId` | `provider == "gcp" AND authMethod == "federation"` | OPTIONAL | +| `gcpRegProviderId` | `provider == "gcp" AND authMethod == "federation"` | OPTIONAL | +| `gcpRegSAEmail` | `provider == "gcp" AND authMethod == "federation"` | OPTIONAL | +| `gcpRegion` | `provider == "gcp"` | OPTIONAL | +| `azureTenantId` | `provider == "azure"` | OPTIONAL | +| `azureACRResource` | `provider == "azure"` | OPTIONAL | +| `azureACRName` | `provider == "azure"` (required for ACR) | **REQUIRED** | +| `azureArtifactsResource` | `provider == "azure"` | OPTIONAL | + +**Valid `authMethod` values per `provider`:** + +| Provider | Valid authMethod values | +|---------------|----------------------------------------------| +| `nexus` | `user_pass`, `anonymous` | +| `artifactory` | `user_pass`, `anonymous` | +| `aws` | `secret`, `assume_role`, `anonymous` | +| `gcp` | `federation`, `service_account`, `anonymous` | +| `azure` | `oauth2`, `anonymous` | + **Examples of different auth sections**: ```yaml @@ -2174,8 +2636,13 @@ authConfig: helm-nexus: authType: longLived + provider: nexus authMethod: user_pass credentialsId: cred-nexus + + docker-anonymous: + provider: nexus + authMethod: anonymous ``` **Example:** @@ -2194,8 +2661,12 @@ authConfig: awsRoleARN: arn:aws:iam::123456789012:role/YourRole helm: authType: longLived + provider: nexus authMethod: user_pass credentialsId: cred-nexus + public-repo: + provider: nexus + authMethod: anonymous mavenConfig: authConfig: aws repositoryDomainName: https://codeartifact.eu-west-1.amazonaws.com/maven/app @@ -2227,15 +2698,18 @@ helmAppConfig: helmReleaseRepoName: helm-releases helmGroupRepoName: helm-group goConfig: + authConfig: public-repo repositoryDomainName: https://nexus.mycompany.internal/repository/go goTargetSnapshot: go-snapshots goTargetRelease: go-releases goProxyRepository: https://goproxy.internal/go/ npmConfig: + authConfig: public-repo repositoryDomainName: https://mycompany.internal npmTargetSnapshot: npm-snapshots npmTargetRelease: npm-releases rawConfig: + authConfig: public-repo repositoryDomainName: https://proxy.raw.local/raw rawTargetSnapshot: raw/snapshots rawTargetRelease: raw/releases @@ -2243,7 +2717,7 @@ rawConfig: rawTargetProxy: https://proxy.raw.local/ ``` -[Registry Definition v2.0 JSON schema](/schemas/regdef-v2.schema.json) +**[Registry Definition v2.0](/python/envgene/envgenehelper/schemas/regdef-v2.schema.json) JSON schema** — bundled in `envgenehelper` package at `python/envgene/envgenehelper/schemas/regdef-v2.schema.json` ### Application Definition diff --git a/docs/envgene-pipelines.md b/docs/envgene-pipelines.md index 35971e4d4..05e66346a 100644 --- a/docs/envgene-pipelines.md +++ b/docs/envgene-pipelines.md @@ -4,7 +4,7 @@ This document describes the CI/CD pipelines and jobs in these pipelines used in Jobs are executed sequentially in the **listed order**. Depending on the condition, a job may or may not be executed. If any job in the sequence fails, the subsequent jobs in the flow are not executed. -The conditions for job execution, the sequence, and the Docker images are the same for both GitLab and GitHub CI/CD platforms. +The conditions for job execution, the sequence, and the Docker images are intended to match across GitLab and GitHub CI/CD platforms. > [!NOTE] > This is the sequence of the core EnvGene. EnvGene is extensible: in extensions, the sequence may be changed and new jobs may be added. @@ -17,30 +17,47 @@ This pipeline is triggered manually by the user via the GitLab/GitHub UI or by a If multiple [`ENV_NAMES`](/docs/instance-pipeline-parameters.md#env_names) are specified: -- For each cluster from `ENV_NAMES`, parallel and independent Cloud Passport discovery flows are started, consisting of (`trigger_passport_job`, `get_passport_job`, `process_decryption_mode_job`). -- For each Environment from `ENV_NAMES`, parallel flows of the remaining jobs are started. +- For each distinct cluster name in `ENV_NAMES`, at most one Cloud Passport flow runs: `trigger_passport_job`, `get_passport_job` (deduplicated by cluster). +- For each environment from `ENV_NAMES`, parallel flows of the remaining jobs are started. ### [Instance pipeline] Job sequence ```mermaid -flowchart LR - A[trigger_passport] --> B[get_passport] --> C[process_decryption_mode] --> D[env_inventory_generation] --> E[credential_rotation] --> F[app_reg_def_process] --> G[process_sd] --> H[env_build] --> I[generate_effective_set] --> J[git_commit] +flowchart TB + subgraph passport["Per-cluster jobs"] + A[trigger_passport] --> B[get_passport] + end + subgraph per_env["Per-environment jobs"] + C[bg_manage] --> D[env_inventory_generation] + D --> E[credential_rotation] + E --> F[app_reg_def_process] + F --> G[process_sd] + G --> H[env_build] + H --> I[generate_effective_set] + I --> J[git_commit] + J --> K[cmdb_import] + end + B --> C ``` -1. **bg_manage** - - **Condition**: Runs if [`BG_MANAGE: true`](/docs/instance-pipeline-parameters.md#bg_manage). - - **Docker image**: [`qubership-envgene`](https://github.com/Netcracker/qubership-envgene/pkgs/container/qubership-envgene) - -2. **trigger_passport**: +1. **trigger_passport**: - **Condition**: Runs if [`GET_PASSPORT: true`](/docs/instance-pipeline-parameters.md#get_passport) - **Docker image**: None. The Discovery repository is triggered from the pipeline -3. **get_passport**: +2. **get_passport**: - **Condition**: Runs if [`GET_PASSPORT: true`](/docs/instance-pipeline-parameters.md#get_passport) - **Docker image**: [`qubership-envgene`](https://github.com/Netcracker/qubership-envgene/pkgs/container/qubership-envgene) +3. **bg_manage** + - **Condition**: Runs if [`BG_MANAGE: true`](/docs/instance-pipeline-parameters.md#bg_manage). + - **Docker image**: [`qubership-envgene`](https://github.com/Netcracker/qubership-envgene/pkgs/container/qubership-envgene) + 4. **env_inventory_generation**: - - **Condition**: Runs if [`ENV_TEMPLATE_TEST: false`](/docs/instance-pipeline-parameters.md#env_template_test) AND ([`ENV_SPECIFIC_PARAMS`](/docs/instance-pipeline-parameters.md#env_specific_params) OR [`ENV_TEMPLATE_NAME`](/docs/instance-pipeline-parameters.md#env_template_name)) + - **Condition**: Runs if [`ENV_TEMPLATE_TEST: false`](/docs/envgene-repository-variables.md#env_template_test) AND any of the following holds: + - [`ENV_INVENTORY_CONTENT`](/docs/instance-pipeline-parameters.md#env_inventory_content) is set, or + - [`ENV_INVENTORY_INIT`](/docs/instance-pipeline-parameters.md#env_inventory_init) is `true`, or + - [`ENV_SPECIFIC_PARAMS`](/docs/instance-pipeline-parameters.md#env_specific_params) is set (non-empty), or + - [`ENV_TEMPLATE_NAME`](/docs/instance-pipeline-parameters.md#env_template_name) is set (non-empty) - **Docker image**: [`qubership-envgene`](https://github.com/Netcracker/qubership-envgene/pkgs/container/qubership-envgene) 5. **credential_rotation**: @@ -50,28 +67,26 @@ flowchart LR 6. **app_reg_def_process**: - **What happens in this job**: 1. Handles certificate updates from the configuration directory. - 2. Downloads the Environment Template artifact. - 3. Renders [Application Definitions](/docs/envgene-objects.md#application-definition) and [Registry Definitions](/docs/envgene-objects.md#registry-definition) from: + 2. Renders [Application Definitions](/docs/envgene-objects.md#application-definition) and [Registry Definitions](/docs/envgene-objects.md#registry-definition) from: 1. Templates, as described in [User Defined by Template](/docs/features/app-reg-defs.md#user-defined-by-template) 2. External Job, as described in [External Job](/docs/features/app-reg-defs.md#external-job) - 4. Runs [Application and Registry Definitions Transformation](/docs/features/app-reg-defs.md#application-and-registry-definitions-transformation) - - **Condition**: Runs if ( [`ENV_BUILD: true`](/docs/instance-pipeline-parameters.md#env_builder) ) + 3. Runs [Application and Registry Definitions Transformation](/docs/features/app-reg-defs.md#application-and-registry-definitions-transformation) + - **Condition**: Runs if [`ENV_BUILD: true`](/docs/instance-pipeline-parameters.md#env_builder) - **Docker image**: [`qubership-envgene`](https://github.com/Netcracker/qubership-envgene/pkgs/container/qubership-envgene) 7. **process_sd**: - - **Condition**: Runs if ( [`SOURCE_TYPE: json`](/docs/instance-pipeline-parameters.md#sd_source_type) AND [`SD_DATA`](/docs/instance-pipeline-parameters.md#sd_data) is provided ) OR ( [`SOURCE_TYPE: artifact`](/docs/instance-pipeline-parameters.md#sd_source_type) AND [`SD_VERSIONS`](/docs/instance-pipeline-parameters.md#sd_version) is provided ) + - **Condition**: Runs if ( [`SD_SOURCE_TYPE: json`](/docs/instance-pipeline-parameters.md#sd_source_type) AND [`SD_DATA`](/docs/instance-pipeline-parameters.md#sd_data) is provided ) OR ( [`SD_SOURCE_TYPE: artifact`](/docs/instance-pipeline-parameters.md#sd_source_type) AND [`SD_VERSION`](/docs/instance-pipeline-parameters.md#sd_version) is provided ) - **Docker image**: [`qubership-envgene`](https://github.com/Netcracker/qubership-envgene/pkgs/container/qubership-envgene) 8. **env_build**: - **What happens in this job**: 1. Handles certificate updates from the configuration directory. - 2. Downloads the Environment Template artifact from the `app_reg_def_process` job artifacts, **not from the registry** - 3. Updates the Environment Template version if [`ENV_TEMPLATE_VERSION`](/docs/instance-pipeline-parameters.md#env_template_version) is provided. - 4. Renders the environment using Jinja2 templates (renders Namespaces, Clouds, and other environment components, but not Application and Registry Definitions). - 5. Handles template overrides - 6. Handles template Parameter Set and Resource profiles. - 7. Handles environment-specific Parameter Set and Resource profiles. - 8. Creates Credentials including shared Credentials + 2. Updates the Environment Template version if [`ENV_TEMPLATE_VERSION`](/docs/instance-pipeline-parameters.md#env_template_version) is provided. + 3. Renders the environment using Jinja2 templates (renders Namespaces, Clouds, and other environment components, but not Application and Registry Definitions). + 4. Handles template overrides + 5. Handles template Parameter Set and Resource profiles. + 6. Handles environment-specific Parameter Set and Resource profiles. + 7. Creates Credentials including shared Credentials - **Condition**: Runs if [`ENV_BUILD: true`](/docs/instance-pipeline-parameters.md#env_builder). - **Docker image**: [`qubership-envgene`](https://github.com/Netcracker/qubership-envgene/pkgs/container/qubership-envgene) @@ -80,5 +95,11 @@ flowchart LR - **Docker image**: [`qubership-effective-set-generator`](https://github.com/Netcracker/qubership-envgene/pkgs/container/qubership-effective-set-generator) 10. **git_commit**: - - **Condition**: Runs if there are jobs requiring changes to the repository AND [`ENV_TEMPLATE_TEST: false`](/docs/instance-pipeline-parameters.md#env_template_test) + - **Condition**: Runs if there are jobs requiring changes to the repository AND [`ENV_TEMPLATE_TEST: false`](/docs/envgene-repository-variables.md#env_template_test) - **Docker image**: [`qubership-envgene`](https://github.com/Netcracker/qubership-envgene/pkgs/container/qubership-envgene) + +11. **cmdb_import**: + - **Condition**: Runs if [`CMDB_IMPORT: true`](/docs/instance-pipeline-parameters.md#cmdb_import) + + > [!NOTE] + > The `cmdb_import` job is **not** part of core EnvGene. It is an **extension point** diff --git a/docs/envgene-repository-variables.md b/docs/envgene-repository-variables.md index c8d84579d..f47da8214 100644 --- a/docs/envgene-repository-variables.md +++ b/docs/envgene-repository-variables.md @@ -13,7 +13,10 @@ - [`GH_RUNNER_TAG_NAME`](#gh_runner_tag_name) - [`RUNNER_SCRIPT_TIMEOUT`](#runner_script_timeout) - [`GH_RUNNER_SCRIPT_TIMEOUT`](#gh_runner_script_timeout) + - [`CALCULATOR_CLI_JAVA_OPTIONS`](#calculator_cli_java_options) - [`DOCKER_REGISTRY` (in instance repository)](#docker_registry-in-instance-repository) + - [`DOCKER_CLOUD_REGISTRY_PROVIDER`](#docker_cloud_registry_provider) + - [`GCP_SA_KEY`](#gcp_sa_key) - [Template EnvGene Repository](#template-envgene-repository) - [`ENV_TEMPLATE_TEST`](#env_template_test) - [`ENVGENE_LOG_LEVEL` (in template repository)](#envgene_log_level-in-template-repository) @@ -141,6 +144,20 @@ This parameter is only available in the GitHub version of the pipeline. For more **Example**: `15` +### `CALCULATOR_CLI_JAVA_OPTIONS` + +**Description**: Java options passed to the Calculator CLI to override default settings. Used to control heap size and ForkJoinPool thread count (number of applications processed in parallel during effective set generation). + +**Default Value**: None + +**Mandatory**: No + +**Example**: + +```text +CALCULATOR_CLI_JAVA_OPTIONS="-Djava.util.concurrent.ForkJoinPool.common.parallelism=4 -Xmx2g -Xms2g" +``` + ### `DOCKER_REGISTRY` (in instance repository) **Description**: Specifies the registry where the EnvGene Docker images are located @@ -151,6 +168,32 @@ This parameter is only available in the GitHub version of the pipeline. For more **Example**: `registry.example.com/docker` +### `DOCKER_CLOUD_REGISTRY_PROVIDER` + +**Description**: Cloud provider for Docker registry authentication when pulling EnvGene Docker images. Currently, the only supported value is `GCP`. When set to `GCP`, the GitHub workflow authenticates to Google Artifact Registry (GAR) before pulling EnvGene images. Used together with [`DOCKER_REGISTRY`](#docker_registry-in-instance-repository) and [`GCP_SA_KEY`](#gcp_sa_key). + +**Default Value**: None + +**Mandatory**: No + +**Allowed Values**: `GCP` (only) + +**Example**: `GCP` + +**Note**: This parameter is used only in the GitHub EnvGene pipeline. For GitLab, use runner-level configuration. See [Docker Registry Configuration](/docs/how-to/docker-registry-configuration.md) for details. + +### `GCP_SA_KEY` + +**Description**: Full JSON content of the GCP service account key. Used for authenticating to Google Artifact Registry (GAR) when pulling EnvGene Docker images. Required only when [`DOCKER_CLOUD_REGISTRY_PROVIDER`](#docker_cloud_registry_provider) is set to `GCP`. + +**Default Value**: None + +**Mandatory**: No (required only for GAR authentication) + +**Example**: `{"type":"service_account","project_id":"...",...}` + +**Note**: Store as a secret (GitHub Actions Secrets) or masked variable. Never commit to the repository. Use a service account with at least `Artifact Registry Reader` role. See [Docker Registry Configuration](/docs/how-to/docker-registry-configuration.md) for details. + ## Template EnvGene Repository ### `ENV_TEMPLATE_TEST` diff --git a/docs/features/app-reg-defs.md b/docs/features/app-reg-defs.md index 52f141ad4..de6f42d13 100644 --- a/docs/features/app-reg-defs.md +++ b/docs/features/app-reg-defs.md @@ -46,6 +46,9 @@ There are two sources for obtaining Application and Registry Definitions in EnvG #### External Job +> [!WARNING] +> The External Job–based mechanism is **deprecated**, is not recommended for use in new or actively maintained environments, and is planned to be removed in a future EnvGene release. Consumers should migrate to template-based Application and Registry Definitions as soon as reasonably possible. + An external job (not implemented in EnvGene itself, but serves as an extension point) that somehow creates/discovers/generates Application and Registry Definitions as YAML files and saves them in its artifact with the contract name `definitions.zip`. During the [`app_reg_def_process`](/docs/envgene-pipelines.md#instance-pipeline) job execution, EnvGene retrieves the Application and Registry Definitions from this artifact and saves them as part of the Environment instance. diff --git a/docs/features/blue-green-deployment.md b/docs/features/blue-green-deployment.md index c593e477a..4d3d51759 100644 --- a/docs/features/blue-green-deployment.md +++ b/docs/features/blue-green-deployment.md @@ -81,15 +81,15 @@ The following functionality is used in these scenarios: - EnvGene generates a [BG Domain](/docs/envgene-objects.md#bg-domain) from a [BG Domain Template](/docs/envgene-objects.md#bg-domain-template) as part of Environment Instance generation - EnvGene validates that namespaces referenced in the BG Domain object exist in the Environment during Environment Instance generation - EnvGene is able to generate particular [Namespaces](/docs/envgene-objects.md#namespace) only of Environment using [Namespace Render Filter](#namespace-render-filter) feature -- EnvGene provides parameters describing BG domain in [Effective Set](/docs/calculator-cli.md#version-20topology-context-bg_domain-example) +- EnvGene provides parameters describing BG domain in [Effective Set](/docs/features/calculator-cli.md#version-20topology-context-bg_domain-example) - EnvGene creates, updates and validates [BG state files](#bg-state-files) for peer and origin namespaces, based on BG Plugin call -- EnvGene supports the [warmup operation](#warmup-operation) by copying [Namespace](https://github.com/Netcracker/qubership-envgene/blob/main/docs/envgene-objects.md#namespace) and [Application](https://github.com/Netcracker/qubership-envgene/blob/main/docs/envgene-objects.md#application) for origin/peer +- EnvGene supports the [warmup operation](#warmup-operation) by copying [Namespace](/docs/envgene-objects.md#namespace) and [Application](/docs/envgene-objects.md#application) for origin/peer - EnvGene [imports](#cmdb-import) the BG domain object into CMDB ### BG Related EnvGene objects -- [BG Domain](https://github.com/Netcracker/qubership-envgene/blob/main/docs/envgene-objects.md#bg-domain): Configuration object that defines domain structure -- [BG Domain Template](https://github.com/Netcracker/qubership-envgene/blob/main/docs/envgene-objects.md#bg-domain-template): Template used to generate BG Domain object during Environment Instance generation. During Environment Instance generation, EnvGene validates that all namespaces referenced in the generated BG Domain object (origin, peer, and controller namespaces) actually exist in the Environment. If any referenced namespace is missing, the generation fails with a validation error. +- [BG Domain](/docs/envgene-objects.md#bg-domain): Configuration object that defines domain structure +- [BG Domain Template](/docs/envgene-objects.md#bg-domain-template): Template used to generate BG Domain object during Environment Instance generation. During Environment Instance generation, EnvGene validates that all namespaces referenced in the generated BG Domain object (origin, peer, and controller namespaces) actually exist in the Environment. If any referenced namespace is missing, the generation fails with a validation error. - [BG State Files](/docs/envgene-objects.md#bg-state-files): Files that track origin and peer namespace states ### Namespace Render Filter @@ -105,13 +105,13 @@ This job is part of the Instance pipeline and does the following: - Validates namespace names in `BG_STATE` against the [BG Domain](/docs/envgene-objects.md#bg-domain) object in the Environment Instance - Validates BG states received in `BG_STATE` against BG state files in the repository - Creates/updates [BG state files](/docs/envgene-objects.md#bg-state-files) -- During warmup, copies [Namespace](https://github.com/Netcracker/qubership-envgene/blob/main/docs/envgene-objects.md#namespace) and [Applications](https://github.com/Netcracker/qubership-envgene/blob/main/docs/envgene-objects.md#application) under it +- During warmup, copies [Namespace](/docs/envgene-objects.md#namespace) and [Applications](/docs/envgene-objects.md#application) under it The criteria for running this job and its order relative to other jobs are described in [envgene-pipelines](/docs/envgene-pipelines.md). ### BG Related Instance Pipeline Parameters -- [`ENV_NAMES`](https://github.com/Netcracker/qubership-envgene/blob/main/docs/instance-pipeline-parameters.md#env_names) +- [`ENV_NAMES`](/docs/instance-pipeline-parameters.md#env_names) - [`BG_MANAGE`](/docs/instance-pipeline-parameters.md#bg_manage) - [`BG_STATE`](/docs/instance-pipeline-parameters.md#bg_state) - [`GH_ADDITIONAL_PARAMS`](/docs/instance-pipeline-parameters.md#gh_additional_params) @@ -235,7 +235,7 @@ This ensures that the candidate namespace will use the same template artifact ve ### CMDB Import -The CMDB Import feature creates, among other entities such as Cloud or Namespace the [Blue Green Domain](https://github.com/Netcracker/qubership-envgene/blob/main/docs/envgene-objects.md#bg-domain) in the CMDB. +The CMDB Import feature creates, among other entities such as Cloud or Namespace the [Blue Green Domain](/docs/envgene-objects.md#bg-domain) in the CMDB. To do this, run the instance pipeline with the `CMDB_IMPORT: true` pipeline parameter. diff --git a/docs/features/calculator-cli.md b/docs/features/calculator-cli.md index 81a72ee29..28aeac12c 100644 --- a/docs/features/calculator-cli.md +++ b/docs/features/calculator-cli.md @@ -35,6 +35,7 @@ - [\[Version 2.0\]\[Deployment Parameter Context\] `credentials.yaml`](#version-20deployment-parameter-context-credentialsyaml) - [\[Version 2.0\] Predefined `credentials.yaml` parameters](#version-20-predefined-credentialsyaml-parameters) - [\[Version 2.0\]\[Deployment Parameter Context\] Collision Parameters](#version-20deployment-parameter-context-collision-parameters) + - [\[Version 2.0\]\[Deployment Parameter Context\] `custom-params.yaml`](#version-20deployment-parameter-context-custom-paramsyaml) - [\[Version 2.0\]\[Deployment Parameter Context\] `deploy-descriptor.yaml`](#version-20deployment-parameter-context-deploy-descriptoryaml) - [\[Version 2.0\] Predefined `deploy-descriptor.yaml` parameters](#version-20-predefined-deploy-descriptoryaml-parameters) - [\[Version 2.0\] Service Artifacts](#version-20-service-artifacts) @@ -72,7 +73,7 @@ 2. Calculator command-line tool must support [Effective Set version 2.0](#effective-set-v20) generation 3. Calculator command-line tool must process [execution attributes](#calculator-command-line-tool-execution-attributes) 4. Calculator command-line tool must not encrypt or decrypt sensitive parameters (credentials.yaml) -5. Calculator command-line tool must resolve [macros](/docs/template-macros.md#calculator-cli-macros) +5. Calculator command-line tool must resolve [macros](/docs/template-macros.md#calculator-command-line-tool-macros) 6. Calculator command-line tool should not process Parameter Sets 7. Calculator command-line tool must not cast parameters type 8. Calculator command-line tool must display reason of error @@ -98,19 +99,20 @@ Below is a **complete** list of attributes -| Attribute | Type | Mandatory | Description | Default | Example | -|-----------------------------------------------------|---------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|--------------------------------------------------------------------------------------| -| `--env-id`/`-e` | string | yes | Environment ID in `/` notation | N/A | `cluster/platform-00` | -| `--envs-path`/`-ep` | string | yes | Path to `/environments` folder | N/A | `/environments` | -| `--sboms-path`/`-sp` | string | no | Path to the folder with Application SBOMs. If the attribute is not provided, generation occurs in [No SBOMs Mode](#version-20-no-sboms-mode) | N/A | `/sboms` | -| `--sd-path`/`-sdp` | string | yes | Path to the Solution Descriptor | N/A | `/environments/cluster/platform-00/Inventory/solution-descriptor/sd.yaml` | -| `--registries`/`-r` | string | no | Required when `--sd-path` and `--sboms-path` are provided. Optional for [No SBOMs Mode](#version-20-no-sboms-mode) | N/A | `/configuration/registry.yml` | -| `--output`/`-o` | string | yes | Folder where the result will be put by Calculator command-line tool | N/A | `/environments/cluster/platform-00/effective-set` | -| `--effective-set-version`/`-esv` | string | no | The version of the effective set to be generated. Available options are `v1.0` and `v2.0` | `v2.0` | `v1.0` | -| `--pipeline-consumer-specific-schema-path`/`-pcssp` | string | no | Path to a JSON schema defining a consumer-specific pipeline context component. Multiple attributes of this type can be provided | N/A | | -| `--extra_params`/`-ex` | string | no | Additional parameters used by the Calculator for effective set generation. Multiple instances of this attribute can be provided | N/A | `DEPLOYMENT_SESSION_ID=550e8400-e29b-41d4-a716-446655440000` | -| `--app_chart_validation`/`-acv` | boolean | no | Determines whether [app chart validation](#version-20-app-chart-validation) should be performed. If `true` validation is enabled (checks for `application/vnd.qubership.app.chart` in SBOM). If `false` validation is skipped | `true` | `false` | -| `--enable-traceability`/`-etr` | boolean | no | Determines whether [traceability](#version-20-traceability-comments) will be enabled. If `true`, traceability comments will be added. If `false`, they will be omitted. | `false` | `true` | +| Attribute | Type | Mandatory | Description | Default | Example | +|-----------------------------------------------------|---------|-----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|---------------------------------------------------------------------------| +| `--env-id`/`-e` | string | yes | Environment ID in `/` notation | N/A | `cluster/platform-00` | +| `--envs-path`/`-ep` | string | yes | Path to `/environments` folder | N/A | `/environments` | +| `--sboms-path`/`-sp` | string | no | Path to the folder with Application SBOMs. If the attribute is not provided, generation occurs in [No SBOMs Mode](#version-20-no-sboms-mode) | N/A | `/sboms` | +| `--sd-path`/`-sdp` | string | yes | Path to the Solution Descriptor | N/A | `/environments/cluster/platform-00/Inventory/solution-descriptor/sd.yaml` | +| `--registries`/`-r` | string | no | Required when `--sd-path` and `--sboms-path` are provided. Optional for [No SBOMs Mode](#version-20-no-sboms-mode) | N/A | `/configuration/registry.yml` | +| `--output`/`-o` | string | yes | Folder where the result will be put by Calculator command-line tool | N/A | `/environments/cluster/platform-00/effective-set` | +| `--effective-set-version`/`-esv` | string | no | The version of the effective set to be generated. Available options are `v1.0` and `v2.0` | `v2.0` | `v1.0` | +| `--pipeline-consumer-specific-schema-path`/`-pcssp` | string | no | Path to a JSON schema defining a consumer-specific pipeline context component. Multiple attributes of this type can be provided | N/A | | +| `--extra_params`/`-ex` | string | no | Additional parameters used by the Calculator for effective set generation. Multiple instances of this attribute can be provided | N/A | `DEPLOYMENT_SESSION_ID=550e8400-e29b-41d4-a716-446655440000` | +| `--app_chart_validation`/`-acv` | boolean | no | Determines whether [app chart validation](#version-20-app-chart-validation) should be performed. If `true` validation is enabled (checks for `application/vnd.qubership.app.chart` in SBOM). If `false` validation is skipped | `true` | `false` | +| `--enable-traceability`/`-etr` | boolean | no | Determines whether [traceability](#version-20-traceability-comments) will be enabled. If `true`, traceability comments will be added. If `false`, they will be omitted. | `false` | `true` | +| `--custom-params`/`-cp` | string | no | [Custom Params](/docs/glossary.md#custom-params) to inject into the Effective Set with highest priority. Applied to deployment, runtime, and cleanup contexts. Treated as sensitive. JSON-in-string format; value structure described in [CUSTOM_PARAMS](/docs/instance-pipeline-parameters.md#custom_params) | N/A | `"{\"deployment\":{\"KEY\":\"val\"}}"` | ### Registry Configuration @@ -278,7 +280,8 @@ Effective Set generation in Version 1.0 does not support [No SBOMs Mode](#versio | | | ├── collision-deployment-parameters.yaml | | | ├── credentials.yaml | | | ├── collision-credentials.yaml - | | | └── deploy-descriptor.yaml + | | | ├── deploy-descriptor.yaml + | | | └── custom-params.yaml | | └── | | └── values | | ├── per-service-parameters @@ -292,7 +295,8 @@ Effective Set generation in Version 1.0 does not support [No SBOMs Mode](#versio | | ├── collision-deployment-parameters.yaml | | ├── credentials.yaml | | ├── collision-credentials.yaml - | | └── deploy-descriptor.yaml + | | ├── deploy-descriptor.yaml + | | └── custom-params.yaml | └── | ├── | | └── values @@ -307,7 +311,8 @@ Effective Set generation in Version 1.0 does not support [No SBOMs Mode](#versio | | ├── collision-deployment-parameters.yaml | | ├── credentials.yaml | | ├── collision-credentials.yaml - | | └── deploy-descriptor.yaml + | | ├── deploy-descriptor.yaml + | | └── custom-params.yaml | └── | └── values | ├── per-service-parameters @@ -321,7 +326,8 @@ Effective Set generation in Version 1.0 does not support [No SBOMs Mode](#versio | ├── collision-deployment-parameters.yaml | ├── credentials.yaml | ├── collision-credentials.yaml - | └── deploy-descriptor.yaml + | ├── deploy-descriptor.yaml + | └── custom-params.yaml ├── runtime | ├── mapping.yml | ├── @@ -428,6 +434,7 @@ Sensitive parameters in the Effective Set are grouped into dedicated credentials 4. `effective-set/deployment///credentials.yaml` 5. `effective-set/deployment///collision-credentials.yaml` 6. `effective-set/runtime///credentials.yaml` +7. `effective-set/deployment///values/custom-params.yaml` **Splitting principle:** @@ -488,7 +495,7 @@ These contexts are used as a source exclusively from the environment instance. Parameters in the Effective Set come from different sources: 1. User-defined in the [Environment Instance](/docs/envgene-objects.md#environment-instance-objects) -2. [Application SBOM](/docs/sbom.md#application-sbom) +2. [Application SBOM](/docs/features/sbom.md#application-sbom) 3. Generated by the calculator 4. Calculator extra parameters (`--extra_params`) 5. Calculator defaults @@ -503,26 +510,27 @@ Parameters can be set at different levels: If the same parameter key is set in multiple sources or levels, the Calculator uses the following priority (from highest to lowest): -1. User-defined in [Resource Profile Override](/docs/envgene-objects.md#resource-profile-override-in-instance) of the Environment Instance -2. Service level defined in Resource Profile Baseline in Application SBOM -3. Service level defined in other Application SBOM attributes -4. Calculator-generated at the Service level -5. User-defined at the [Application](/docs/envgene-objects.md#application) level in Environment Instance -6. Application level defined in Application SBOM -7. Calculator-generated at the Application level -8. User-defined at the [Namespace](/docs/envgene-objects.md#namespace) level in Environment Instance -9. Namespace level defined in Application SBOM -10. Calculator-generated at the Namespace level -11. User-defined at the [Cloud](/docs/envgene-objects.md#cloud) level in Environment Instance -12. Cloud level defined in Application SBOM -13. Calculator-generated at the Cloud level -14. User-defined at the [Tenant](/docs/envgene-objects.md#tenant) level in Environment Instance -15. Calculator extra parameters (`--extra_params`) -16. Default values by calculator +1. [Custom Params](/docs/glossary.md#custom-params) (`--custom-params`) +2. User-defined in [Resource Profile Override](/docs/envgene-objects.md#resource-profile-override) of the Environment Instance +3. Service level defined in Resource Profile Baseline in Application SBOM +4. Service level defined in other Application SBOM attributes +5. Calculator-generated at the Service level +6. User-defined at the [Application](/docs/envgene-objects.md#application) level in Environment Instance +7. Application level defined in Application SBOM +8. Calculator-generated at the Application level +9. User-defined at the [Namespace](/docs/envgene-objects.md#namespace) level in Environment Instance +10. Namespace level defined in Application SBOM +11. Calculator-generated at the Namespace level +12. User-defined at the [Cloud](/docs/envgene-objects.md#cloud) level in Environment Instance +13. Cloud level defined in Application SBOM +14. Calculator-generated at the Cloud level +15. User-defined at the [Tenant](/docs/envgene-objects.md#tenant) level in Environment Instance +16. Calculator extra parameters (`--extra_params`) +17. Default values by calculator For example, if a user sets `BASELINE_PROJ` at the Namespace level, this value will override the value calculated by the calculator. If `BASELINE_PROJ` is set at the cloud level, it will be overridden by calculator-generated values at Namespace or Application levels. -Not every parameter in the Effective Set can be overridden. Overriding service-level parameters is only possible via [resource profile override](/docs/envgene-objects.md#resource-profile-override-in-instance). +Not every parameter in the Effective Set can be overridden. Overriding service-level parameters is only possible via [resource profile override](/docs/envgene-objects.md#resource-profile-override). > [!NOTE] > If a complex parameter is set, the value from the higher-priority source **completely replaces** the lower-priority one; merging is not performed. @@ -535,33 +543,34 @@ The CLI flag [`--enable-traceability`](#calculator-command-line-tool-execution-a ##### Parameter Source to Comment Mapping -| Parameter Source | Comment | Example | -|----------------------------------------------------|--------------------------------------------|----------------------------------------------------------------------------------------------| -| Environment Instance, Tenant | `# tenant` | `GITLAB_URL: "https://git.qibership.org" # tenant` | -| Environment Instance, Cloud | `# cloud` | `CLOUD_API_HOST: "https://api.example.com" # cloud` | -| Environment Instance, Namespace | `# namespace: ` | `NAMESPACE_NAME: "env-1-core" # namespace: env-1-core` | -| Environment Instance, Application | `# application: ` | `APP_FEATURE_FLAG: true # application: my-app` | -| Environment Instance, Resource Profile Override | `# resource-profile-override: ` | `CPU_LIMIT: "500m" # resource-profile-override: perf-small` | -| Environment Instance, Composite Structure | `# composite-structure` | `composite_structure: # composite-structure` | -| Environment Instance, BG Domain | `# bg-domain` | `bg_domain: # bg-domain` | -| Application SBOM | `# sbom` | `deploy_param: '' # sbom` | -| Application SBOM, Resource Profile Baseline | `# sbom, resource-profile-baseline: `| `PROFILE_BASELINE: "dev" # sbom, resource-profile-baseline: dev` | -| Calculated by calculator | `# envgene calculated` | `PUBLIC_GATEWAY_URL: "https://public-gateway-bss.qubership.org" # envgene calculated` | -| Calculator `--extra_params` | `# envgene pipeline parameter` | `DEPLOYMENT_SESSION_ID: "7e9f5f54-4be2-4fbd-a267-19e78d09810d" # envgene pipeline parameter` | -| Default value by calculator | `# envgene default` | `MANAGED_BY: "argocd" # envgene default` | +| Parameter Source | Comment | Example | +|-------------------------------------------------|-------------------------------|---------------------------------------------------------------------------------------------| +| Custom Params (`--custom-params`) | `#custom params` | `OVERRIDE_KEY: "value" #custom params` | +| Environment Instance, Tenant | `#tenant` | `GITLAB_URL: "https://git.qibership.org" #tenant` | +| Environment Instance, Cloud | `#cloud` | `CLOUD_API_HOST: "https://api.example.com" #cloud` | +| Environment Instance, Namespace | `#namespace` | `NAMESPACE_NAME: "env-1-core" #namespace` | +| Environment Instance, Application | `#application` | `APP_FEATURE_FLAG: true #application` | +| Environment Instance, Resource Profile Override | `#rp-override: ` | `CPU_LIMIT: "500m" #rp-override: perf-small` | +| Environment Instance, Composite Structure | `#composite-structure` | `composite_structure: {} #composite-structure` | +| Environment Instance, BG Domain | `#bg-domain` | `bg_domain: {} #bg-domain` | +| Application SBOM (component properties) | `#sbom` | `git_revision: "a71c5988" #sbom` | +| Application SBOM, Resource Profile Baseline | `#rp-baseline: ` | `CPU_LIMIT: "500m" #rp-baseline: dev` | +| Calculated by calculator | `#envgene calculated` | `PUBLIC_GATEWAY_URL: "https://public-gateway-bss.qubership.org" #envgene calculated` | +| Calculator `--extra_params` | `#envgene pipeline parameter` | `DEPLOYMENT_SESSION_ID: "7e9f5f54-4be2-4fbd-a267-19e78d09810d" #envgene pipeline parameter` | +| Default value by calculator | `#envgene default` | `MANAGED_BY: "argocd" #envgene default` | ##### Rules for Adding Comments -1. The comment is added after a single space following the parameter value on the same line for non-multiline values. +1. The comment is added immediately after the parameter value (no space between value and `#`) on the same line for non-multiline values. ```yaml - SECURITY_POLICY: strict # cloud + SECURITY_POLICY: strict #cloud ``` 2. The comment is added on the previous line above the parameter for multiline values (using `|` or `>`). ```yaml - # cloud + #cloud CS_CONTENT_SECURITY_POLICY: | {"CONTENT_SECURITY_POLICY":"default-src..."} ``` @@ -580,24 +589,42 @@ The CLI flag [`--enable-traceability`](#calculator-command-line-tool-execution-a ```yaml servers: - - "server1.example.com" # cloud - - "server2.example.com" # namespace: env-1 - + - "server1.example.com" #cloud + - "server2.example.com" #namespace + servers: - name: "server1" # cloud - host: "host1" # cloud - port: 8080 # namespace: env-1 + name: "server1" #cloud + host: "host1" #cloud + port: 8080 #namespace ``` 9. Comments are not added to YAML anchors/aliases (`&id001`, `*id001`, `<<: *id001`). But for regular keys/values filled in via anchors/aliases, still show their source as a comment: ```yaml global: &id001 - key1: value1 # cloud - key2: value2 # cloud + key1: value1 #cloud + key2: value2 #cloud service1: <<: *id001 - key3: value3 # namespace: env-1-core + key3: value3 #namespace + ``` + +10. **Exception for `deploy-descriptor.yaml`** — this file is almost entirely generated from the Application SBOM. As an exception to Rule 8, per-line comments are **not** added. Instead, a single file-level header comment is placed at the top of the file describing the source for all parameters. Parameters whose source differs from the header (e.g. predefined parameters with `#rp-baseline: ` or `#rp-override: `) are still marked inline. + + Example: + + ```yaml + #Source of parameters not marked inline: `#sbom` + global: + deployDescriptor: + my-service: + git_revision: a71c5988fc92de5f9698434bfe43d513245969aa + build_id_dtrust: 3d28937a-c7ce-4600-b454-6c08ae61f557 + service_name: my-service + version: release-2025.3-9.16.0 + type: cr + REPLICAS: 1 #rp-override: dev-override + CPU_LIMIT: 500m #rp-baseline: dev ``` #### [Version 2.0] Deployment Parameter Context @@ -654,19 +681,19 @@ The `` can be complex, such as a map or a list, whose elements can also b | `SERVER_HOSTNAME` | yes | string | **Deprecated**. Uses `CLOUD_PUBLIC_HOST` if set, otherwise falls back to `CLOUD_API_HOST` | None | N/A | | `CUSTOM_HOST` | yes | string | **Deprecated**. Uses `CLOUD_PRIVATE_HOST` if set, otherwise falls back to `SERVER_HOSTNAME` | None | N/A | | `OPENSHIFT_SERVER` | yes | string | **Deprecated**. Constructed as `CLOUD_PROTOCOL`://`CLOUD_PUBLIC_HOST`:`CLOUD_API_PORT` | None | N/A | -| `DBAAS_ENABLED` | yes | boolean | Feature toggle indicating whether DBaaS is used | `false` | `dbaasConfigs[0].enable` in the `Cloud` | -| `API_DBAAS_ADDRESS` | no | string | DBaaS API endpoint accessible within a cluster network. Provided if `DBAAS_ENABLED: true` only | None | `dbaasConfigs[0].apiUrl` in the `Cloud` | -| `DBAAS_AGGREGATOR_ADDRESS` | no | string | DBaaS API endpoint accessible outside the cluster network. Provided if `DBAAS_ENABLED: true` only | None | `dbaasConfigs[0].aggregatorUrl` in the `Cloud` | -| `MAAS_ENABLED` | yes | boolean | Feature toggle indicating whether MaaS is used | `false` | `maasConfig.enable` in the `Cloud` | -| `MAAS_INTERNAL_ADDRESS` | no | string | MaaS API endpoint accessible within a cluster network. Provided if `MAAS_ENABLED: true` only | None | `maasConfig.maasInternalAddress` in the `Cloud` | -| `MAAS_EXTERNAL_ROUTE` | no | string | Maas API endpoint accessible outside the cluster network. Provided if `MAAS_ENABLED: true` only | None | `maasConfig.maasUrl` in the `Cloud` | -| `MAAS_SERVICE_ADDRESS` | no | string | **Deprecated**. The same as `MAAS_EXTERNAL_ROUTE`. Provided if `MAAS_ENABLED: true` only | None | `maasConfig.maasUrl` in the `Cloud` | -| `VAULT_ENABLED` | yes | boolean | Feature toggle indicating whether Vault is used | `false` | `vaultConfig.enable` in the `Cloud` | -| `VAULT_ADDR` | no | string | Vault API endpoint accessible within a cluster network. Provided if `VAULT_ENABLED: true` only | None | `vaultConfig.enable` in the `Cloud` | -| `PUBLIC_VAULT_URL` | no | string | Vault API endpoint accessible outside the cluster network. Provided if `VAULT_ENABLED: true` only | None | `vaultConfig.url` in the `Cloud` | -| `CONSUL_ENABLED` | yes | boolean | Feature toggle indicating whether Consul is used | `false` | `consulConfig.enabled` in the `Cloud` | -| `CONSUL_URL` | no | string | Consul API endpoint accessible within a cluster network. Provided if `CONSUL_ENABLED: true` only | None | `consulConfig.internalUrl` in the `Cloud` | -| `CONSUL_PUBLIC_URL` | no | string | Consul API endpoint accessible within a cluster network. Provided if `CONSUL_ENABLED: true` only | None | `consulConfig.internalUrl` in the `Cloud` | +| `DBAAS_ENABLED` | no | boolean | Feature toggle indicating whether DBaaS is used | `false` | `dbaasConfigs[0].enable` in the `Cloud` | +| `API_DBAAS_ADDRESS` | no | string | DBaaS API endpoint accessible within a cluster network. Omitted if `dbaasConfigs[0].enable: false` | None | `dbaasConfigs[0].apiUrl` in the `Cloud` | +| `DBAAS_AGGREGATOR_ADDRESS` | no | string | DBaaS API endpoint accessible outside the cluster network. Omitted if `dbaasConfigs[0].enable: false` | None | `dbaasConfigs[0].aggregatorUrl` in the `Cloud` | +| `MAAS_ENABLED` | no | boolean | Feature toggle indicating whether MaaS is used | `false` | `maasConfig.enable` in the `Cloud` | +| `MAAS_INTERNAL_ADDRESS` | no | string | MaaS API endpoint accessible within a cluster network. Omitted if `maasConfig.enable: false` | None | `maasConfig.maasInternalAddress` in the `Cloud` | +| `MAAS_EXTERNAL_ROUTE` | no | string | Maas API endpoint accessible outside the cluster network. Omitted if `maasConfig.enable: false` | None | `maasConfig.maasUrl` in the `Cloud` | +| `MAAS_SERVICE_ADDRESS` | no | string | **Deprecated**. The same as `MAAS_EXTERNAL_ROUTE`. Omitted if `maasConfig.enable: false` | None | `maasConfig.maasUrl` in the `Cloud` | +| `VAULT_ENABLED` | no | boolean | Feature toggle indicating whether Vault is used | `false` | `vaultConfig.enable` in the `Cloud` | +| `VAULT_ADDR` | no | string | Vault API endpoint accessible within a cluster network. Omitted if `vaultConfig.enable: false` | None | `vaultConfig.enable` in the `Cloud` | +| `PUBLIC_VAULT_URL` | no | string | Vault API endpoint accessible outside the cluster network. Omitted if `vaultConfig.enable: false` | None | `vaultConfig.url` in the `Cloud` | +| `CONSUL_ENABLED` | no | boolean | Feature toggle indicating whether Consul is used | `false` | `consulConfig.enabled` in the `Cloud` | +| `CONSUL_URL` | no | string | Consul API endpoint accessible within a cluster network. Omitted if `consulConfig.enabled: false` | None | `consulConfig.internalUrl` in the `Cloud` | +| `CONSUL_PUBLIC_URL` | no | string | Consul API endpoint accessible within a cluster network. Omitted if `consulConfig.enabled: false` | None | `consulConfig.internalUrl` in the `Cloud` | | `PRODUCTION_MODE` | no | boolean | Defines the deployment environment (non-production/production) type for restricting Helm chart content | `false` | `deployParameters.PRODUCTION_MODE` in the `Cloud` | | `TENANTNAME` | yes | string | Tenant name | None | `name` in the `Tenant` | | `CLOUDNAME` | yes | string | Cloud name | None | `name` in the `Cloud` | @@ -734,14 +761,14 @@ global: &id001 | Attribute | Mandatory | Type | Description | Default | Source Environment Instance | |------------------------------------------|-----------|--------|-------------------------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `K8S_TOKEN` | yes | string | Cluster's API token | None | Taken from `data.secret` in the `Credential` set by `defaultCredentialsId` in the related `Namespace` or parent `Cloud`. If not set in `Namespace`, inherited from `Cloud`. `Namespace` has priority if both are set. | -| `DBAAS_AGGREGATOR_USERNAME` | no | string | DBaaS username | None | Taken from `data.username` property of the `Credential` specified via `dbaasConfigs[0].credentialsId.username` in the `Cloud` | -| `DBAAS_AGGREGATOR_PASSWORD` | no | string | DBaaS password | None | Taken from `data.password` property of the `Credential` specified via `dbaasConfigs[0].credentialsId.password` in the `Cloud` | -| `DBAAS_CLUSTER_DBA_CREDENTIALS_USERNAME` | no | string | same as `DBAAS_AGGREGATOR_USERNAME` | None | Taken from `data.username` property of the `Credential` specified via `dbaasConfigs[0].credentialsId.username` in the `Cloud` | -| `DBAAS_CLUSTER_DBA_CREDENTIALS_PASSWORD` | no | string | same as `DBAAS_AGGREGATOR_PASSWORD` | None | Taken from `data.password` property of the `Credential` specified via `dbaasConfigs[0].credentialsId.password` in the `Cloud` | -| `MAAS_CREDENTIALS_USERNAME` | no | string | MaaS username | None | Taken from `data.username` property of the `Credential` specified via `maasConfig.credentialsId.username` in the `Cloud` | -| `MAAS_CREDENTIALS_PASSWORD` | no | string | MaaS password | None | Taken from `data.password` property of the `Credential` specified via `maasConfig.credentialsId.password` in the `Cloud` | -| `VAULT_TOKEN` | no | string | Vault token | None | Taken from `data.secret` property of the `Credential` specified via `vaultConfig.credentialsId.secret` in the `Cloud` | -| `CONSUL_ADMIN_TOKEN` | no | string | Consul admin token | None | Taken from `data.secret` property of the `Credential` specified via `consulConfig.internalUrl` in the `Cloud` | +| `DBAAS_AGGREGATOR_USERNAME` | no | string | DBaaS username. Omitted if `dbaasConfigs[0].enable: false` or `dbaasConfigs[0].credentialsId: ""` | None | Taken from `data.username` property of the `Credential` specified via `dbaasConfigs[0].credentialsId.username` in the `Cloud` | +| `DBAAS_AGGREGATOR_PASSWORD` | no | string | DBaaS password. Omitted if `dbaasConfigs[0].enable: false` or `dbaasConfigs[0].credentialsId: ""` | None | Taken from `data.password` property of the `Credential` specified via `dbaasConfigs[0].credentialsId.password` in the `Cloud` | +| `DBAAS_CLUSTER_DBA_CREDENTIALS_USERNAME` | no | string | same as `DBAAS_AGGREGATOR_USERNAME`. Omitted if `dbaasConfigs[0].enable: false` or `dbaasConfigs[0].credentialsId: ""` | None | Taken from `data.username` property of the `Credential` specified via `dbaasConfigs[0].credentialsId.username` in the `Cloud` | +| `DBAAS_CLUSTER_DBA_CREDENTIALS_PASSWORD` | no | string | same as `DBAAS_AGGREGATOR_PASSWORD`. Omitted if `dbaasConfigs[0].enable: false` or `dbaasConfigs[0].credentialsId: ""` | None | Taken from `data.password` property of the `Credential` specified via `dbaasConfigs[0].credentialsId.password` in the `Cloud` | +| `MAAS_CREDENTIALS_USERNAME` | no | string | MaaS username. Omitted if `maasConfig.enable: false` or `maasConfig.credentialsId: ""` | None | Taken from `data.username` property of the `Credential` specified via `maasConfig.credentialsId.username` in the `Cloud` | +| `MAAS_CREDENTIALS_PASSWORD` | no | string | MaaS password. Omitted if `maasConfig.enable: false` or `maasConfig.credentialsId: ""` | None | Taken from `data.password` property of the `Credential` specified via `maasConfig.credentialsId.password` in the `Cloud` | +| `VAULT_TOKEN` | no | string | Vault token. Omitted if `vaultConfig.enable: false` or `vaultConfig.credentialsId: ""` | None | Taken from `data.secret` property of the `Credential` specified via `vaultConfig.credentialsId.secret` in the `Cloud` | +| `CONSUL_ADMIN_TOKEN` | no | string | Consul admin token. Omitted if `consulConfig.enabled: false` or `consulConfig.tokenSecret: ""` | None | Taken from `data.secret` property of the `Credential` specified via `consulConfig.tokenSecret` in the `Cloud` | | `SSL_SECRET_VALUE` | no | string | SSL Certificate bundle | None | The value is taken from the deployment parameter `DEFAULT_SSL_CERTIFICATES_BUNDLE`, which can be set at the `Tenant`, `Cloud`, `Namespace`, or `Application` | | `CA_BUNDLE_CERTIFICATE` | no | string | SSL Certificate bundle | None | The value is taken from the deployment parameter `DEFAULT_SSL_CERTIFICATES_BUNDLE`, which can be set at the `Tenant`, `Cloud`, `Namespace`, or `Application` | @@ -773,6 +800,14 @@ The structure of both files is following: These files must only contain keys that match the name of a [services](#version-20-service-inclusion-criteria-and-naming-convention) +##### \[Version 2.0][Deployment Parameter Context] `custom-params.yaml` + +This file is based on the parameter passed to the Calculator via `--custom-params`. + +It contains parameters from the `deployment` section of the object passed to `--custom-params`. + +If `--custom-params` is not passed, the file is generated empty. + ##### \[Version 2.0][Deployment Parameter Context] `deploy-descriptor.yaml` This file describes the parameters of the application artifacts generated during the build process. These parameters are extracted from the Application's SBOM. The file contains a **predefined** set of parameters, and users cannot modify it. @@ -1335,9 +1370,13 @@ The `` can be complex, such as a map or a list, whose elements can also b ##### \[Version 2.0][Runtime Parameter Context] `credentials.yaml` -This file contains sensitive parameters defined in the `technicalConfigurationParameters` section. +This file contains: -For more information, refer to [Sensitive parameters processing](#version-20-sensitive-parameters-processing). +1. Sensitive parameters defined in the `technicalConfigurationParameters` section. For more information, refer to [Sensitive parameters processing](#version-20-sensitive-parameters-processing) + +2. Parameters from the `runtime` section of the object passed to `--custom-params` + +Parameters from `--custom-params` have higher priority. The structure of this file is as follows: @@ -1371,9 +1410,13 @@ The structure of this file is as follows: ##### \[Version 2.0][Cleanup Context] `credentials.yaml` -This file contains sensitive parameters defined in the `deployParameters` sections of the `Tenant`, `Cloud`, and `Namespace` Environment Instance objects. +This file contains -For more information, refer to [Sensitive parameters processing](#version-20-sensitive-parameters-processing). +1. Sensitive parameters defined in the `deployParameters` sections of the `Tenant`, `Cloud`, and `Namespace` Environment Instance objects. For more information, refer to [Sensitive parameters processing](#version-20-sensitive-parameters-processing) + +2. Parameters from the `runtime` section of the object passed to `--custom-params` + +Parameters from `--custom-params` have higher priority. The structure of this file is as follows: diff --git a/docs/features/env-inventory-generation.md b/docs/features/env-inventory-generation.md index d37df4cd3..4f3e3a31c 100644 --- a/docs/features/env-inventory-generation.md +++ b/docs/features/env-inventory-generation.md @@ -92,6 +92,7 @@ The generated Environment Inventory must be reused by other jobs in the same pip | `credentials[].action` | enum [`create_or_replace`, `delete`] | yes | Operation mode for the Shared Credentials file. See [Actions](#actions) | `create_or_replace` | | `credentials[].place` | enum[`site`,`cluster`,`env`] | yes | Defines where the Shared Credentials file is stored. See [Paths by place](#paths-by-place) | `site` | | `credentials[].content` | hashmap | no | Shared Credential as file content. Must be valid according [schema](/schemas/credential.schema.json) | See [example below](#full-env_inventory_content-example) | +| `credentials[].name` | string | yes | Name of the shared credentials file. The file will be saved as `.yml` | | | `resourceProfiles` | array | no | List of Resource Profile Override operations | See [example below](#full-env_inventory_content-example) | | `resourceProfiles[].action` | enum [`create_or_replace`, `delete`] | yes | Operation mode for the Resource Profile Override file. See [Actions](#actions) | `create_or_replace` | | `resourceProfiles[].place` | enum[`site`,`cluster`,`env`] | yes | Defines where the Resource Profile Override file is stored. See [Paths by place](#paths-by-place) | `cluster` | @@ -225,6 +226,7 @@ This example shows how to generate a new Environment Inventory (`env_definition. { "action": "create_or_replace", "place": "site", + "name": "prod-integration-creds", "content": { "prod-integration-creds": { "type": "", @@ -300,7 +302,7 @@ This example shows how to generate a new Environment Inventory (`env_definition. ##### ENV_INVENTORY_CONTENT in JSON-in-string format ```json -"{\"envDefinition\":{\"action\":\"create_or_replace\",\"content\":{\"inventory\":{\"environmentName\":\"env-1\",\"tenantName\":\"Applications\",\"cloudName\":\"cluster-1\",\"description\":\"Fullsample\",\"owners\":\"Qubershipteam\",\"config\":{\"updateRPOverrideNameWithEnvName\":false,\"updateCredIdsWithEnvName\":true}},\"envTemplate\":{\"name\":\"composite-prod\",\"artifact\":\"project-env-template:master_20231024-080204\",\"additionalTemplateVariables\":{\"ci\":{\"CI_PARAM_1\":\"ci-param-val-1\",\"CI_PARAM_2\":\"ci-param-val-2\"},\"e2eParameters\":{\"E2E_PARAM_1\":\"e2e-param-val-1\",\"E2E_PARAM_2\":\"e2e-param-val-2\"}},\"sharedTemplateVariables\":[\"prod-template-variables\",\"sample-cloud-template-variables\"],\"envSpecificParamsets\":{\"bss\":[\"env-specific-bss\"]},\"envSpecificTechnicalParamsets\":{\"bss\":[\"env-specific-tech\"]},\"envSpecificE2EParamsets\":{\"cloud\":[\"cloud-level-params\"]},\"sharedMasterCredentialFiles\":[\"prod-integration-creds\"],\"envSpecificResourceProfiles\":{\"cloud\":[\"cloud-specific-profile\"]}}}},\"paramSets\":[{\"action\":\"create_or_replace\",\"place\":\"env\",\"content\":{\"version\":\"\",\"name\":\"env-specific-bss\",\"parameters\":{\"key\":\"value\"},\"applications\":[]}}],\"credentials\":[{\"action\":\"create_or_replace\",\"place\":\"site\",\"content\":{\"prod-integration-creds\":{\"type\":\"\",\"data\":{\"username\":\"\",\"password\":\"\"}}}}],\"resourceProfiles\":[{\"action\":\"create_or_replace\",\"place\":\"cluster\",\"content\":{\"name\":\"cloud-specific-profile\",\"baseline\":\"dev\",\"description\":\"\",\"applications\":[{\"name\":\"core\",\"version\":\"release-20241103.225817\",\"sd\":\"\",\"services\":[{\"name\":\"operator\",\"parameters\":[{\"name\":\"GATEWAY_MEMORY_LIMIT\",\"value\":\"96Mi\"},{\"name\":\"GATEWAY_CPU_REQUEST\",\"value\":\"50m\"}]}]}],\"version\":0}}],\"sharedTemplateVariables\":[{\"action\":\"create_or_replace\",\"place\":\"site\",\"name\":\"prod-template-variables\",\"content\":{\"TEMPLATE_VAR_1\":\"prod-value-1\",\"TEMPLATE_VAR_2\":\"prod-value-2\",\"nested\":{\"key1\":\"nested-prod-value-1\",\"key2\":\"nested-prod-value-2\"}}},{\"action\":\"create_or_replace\",\"place\":\"cluster\",\"name\":\"sample-cloud-template-variables\",\"content\":{\"CLOUD_VAR_1\":\"cloud-value-1\",\"CLOUD_VAR_2\":\"cloud-value-2\"}}]}" +"{\"envDefinition\":{\"action\":\"create_or_replace\",\"content\":{\"inventory\":{\"environmentName\":\"env-1\",\"tenantName\":\"Applications\",\"cloudName\":\"cluster-1\",\"description\":\"Fullsample\",\"owners\":\"Qubershipteam\",\"config\":{\"updateRPOverrideNameWithEnvName\":false,\"updateCredIdsWithEnvName\":true}},\"envTemplate\":{\"name\":\"composite-prod\",\"artifact\":\"project-env-template:master_20231024-080204\",\"additionalTemplateVariables\":{\"ci\":{\"CI_PARAM_1\":\"ci-param-val-1\",\"CI_PARAM_2\":\"ci-param-val-2\"},\"e2eParameters\":{\"E2E_PARAM_1\":\"e2e-param-val-1\",\"E2E_PARAM_2\":\"e2e-param-val-2\"}},\"sharedTemplateVariables\":[\"prod-template-variables\",\"sample-cloud-template-variables\"],\"envSpecificParamsets\":{\"bss\":[\"env-specific-bss\"]},\"envSpecificTechnicalParamsets\":{\"bss\":[\"env-specific-tech\"]},\"envSpecificE2EParamsets\":{\"cloud\":[\"cloud-level-params\"]},\"sharedMasterCredentialFiles\":[\"prod-integration-creds\"],\"envSpecificResourceProfiles\":{\"cloud\":[\"cloud-specific-profile\"]}}}},\"paramSets\":[{\"action\":\"create_or_replace\",\"place\":\"env\",\"content\":{\"version\":\"\",\"name\":\"env-specific-bss\",\"parameters\":{\"key\":\"value\"},\"applications\":[]}}],\"credentials\":[{\"action\":\"create_or_replace\",\"place\":\"site\",\"name\":\"prod-integration-creds\",\"content\":{\"prod-integration-creds\":{\"type\":\"\",\"data\":{\"username\":\"\",\"password\":\"\"}}}}],\"resourceProfiles\":[{\"action\":\"create_or_replace\",\"place\":\"cluster\",\"content\":{\"name\":\"cloud-specific-profile\",\"baseline\":\"dev\",\"description\":\"\",\"applications\":[{\"name\":\"core\",\"version\":\"release-20241103.225817\",\"sd\":\"\",\"services\":[{\"name\":\"operator\",\"parameters\":[{\"name\":\"GATEWAY_MEMORY_LIMIT\",\"value\":\"96Mi\"},{\"name\":\"GATEWAY_CPU_REQUEST\",\"value\":\"50m\"}]}]}],\"version\":0}}],\"sharedTemplateVariables\":[{\"action\":\"create_or_replace\",\"place\":\"site\",\"name\":\"prod-template-variables\",\"content\":{\"TEMPLATE_VAR_1\":\"prod-value-1\",\"TEMPLATE_VAR_2\":\"prod-value-2\",\"nested\":{\"key1\":\"nested-prod-value-1\",\"key2\":\"nested-prod-value-2\"}}},{\"action\":\"create_or_replace\",\"place\":\"cluster\",\"name\":\"sample-cloud-template-variables\",\"content\":{\"CLOUD_VAR_1\":\"cloud-value-1\",\"CLOUD_VAR_2\":\"cloud-value-2\"}}]}" ``` #### `ENV_SPECIFIC_PARAMS` diff --git a/docs/features/env-specific-schema.md b/docs/features/env-specific-schema.md index 64b47023c..994fb8b85 100644 --- a/docs/features/env-specific-schema.md +++ b/docs/features/env-specific-schema.md @@ -14,7 +14,7 @@ I as DevOps want to define the set of parameters that should be specified by cus ... ``` -2. I'm creating schema for the path (e.g. [env-specific-schema](/docs/samples/template-repository/templates/env_templates/composite/env-specific-schema.yml)) +2. I'm creating the environment-specific schema file at the referenced path. 3. I'm building template diff --git a/docs/features/environment-instance-generation.md b/docs/features/environment-instance-generation.md index 0ce15d4b0..16ad7c94c 100644 --- a/docs/features/environment-instance-generation.md +++ b/docs/features/environment-instance-generation.md @@ -16,7 +16,7 @@ ## Description -This feature describes the process of generating an [Environment Instance](/docs/envgene-objects.md#environment-instance) from an [Environment Template](/docs/envgene-objects.md#environment-template) and [Environment Inventory](/docs/envgene-configs.md#env_definitionyml). The generation process creates the directory structure and files for the Environment Instance, including Namespaces, Applications, Resource Profiles, Credentials, and other EnvGene objects. +This feature describes the process of generating an [Environment Instance](/docs/envgene-objects.md#environment-instance-objects) from an [Environment Template](/docs/envgene-objects.md#environment-template-objects) and [Environment Inventory](/docs/envgene-configs.md#env_definitionyml). The generation process creates the directory structure and files for the Environment Instance, including Namespaces, Applications, Resource Profiles, Credentials, and other EnvGene objects. ## Namespace Folder Name Generation @@ -119,6 +119,7 @@ In this example: ## Related Features -- [Namespace Render Filtering](/docs/features/namespace-render-filtering.md) - Uses namespace folder names for filtering -- [Blue-Green Deployment](/docs/features/blue-green-deployment.md) - Describes BG Domain structure +- [Namespace Render Filter](/docs/features/namespace-render-filtering.md) - Select which Namespaces to render in a specific pipeline run +- [Namespace Filtering in Template Descriptor](/docs/features/namespace-filtering-in-template-descriptor.md) - Filter which Namespaces are included in Environment structure during Template Descriptor rendering +- [Blue-Green Deployment](/docs/features/blue-green-deployment.md) - BG domains and state management - [Effective Set Calculator](/docs/features/calculator-cli.md) - Uses folder names for effective set structure diff --git a/docs/features/namespace-filtering-in-template-descriptor.md b/docs/features/namespace-filtering-in-template-descriptor.md new file mode 100644 index 000000000..a2fc65039 --- /dev/null +++ b/docs/features/namespace-filtering-in-template-descriptor.md @@ -0,0 +1,94 @@ +# Namespace Filtering in Template Descriptor + +- [Namespace Filtering in Template Descriptor](#namespace-filtering-in-template-descriptor) + - [Overview](#overview) + - [Problem Statement](#problem-statement) + - [Proposed Approach](#proposed-approach) + - [Example (cluster type filtering)](#example-cluster-type-filtering) + - [File Resolution Priority](#file-resolution-priority) + - [Behavior During Environment Generation](#behavior-during-environment-generation) + - [Scenario: Generating an Environment with namespace filtering](#scenario-generating-an-environment-with-namespace-filtering) + - [What happens when a namespace condition is `false`](#what-happens-when-a-namespace-condition-is-false) + - [How-To](#how-to) + +## Overview + +Namespace Filtering in Template Descriptor allows generating an Environment that includes only a selected subset of namespaces from a unified Environment Template. + +This feature works during Environment Instance generation, and defines the structural composition of the generated Environment. + +## Problem Statement + +We have been using a unified Environment Template for different projects. This template contains all possible namespaces, but during deployment we often need only a subset of available namespaces. + +Currently, there is no possibility to filter namespaces during Environment Instance generation. This leads to the following: + +- The `Namespaces/` folder in the EnvGene Instance Repository contains non-relevant namespaces +- The Effective Set includes namespaces that are not required for the target deployment +- We cannot include/exclude namespaces specific to the cluster type (k8s or ocp) + +It prevents using a single unified Environment Template + +## Proposed Approach + +Treat the Template Descriptor (TD) as a Jinja template while keeping backward compatibility for non-Jinja TD. + +This enables filtering individual namespaces during Environment generation using Jinja `if` expressions. + +You can use different filtering approaches depending on your use case: + +- **Cluster-type filtering**: Use a variable to include/exclude namespaces based on cluster type (k8s/ocp) +- **Explicit namespace list**: Use shared template variables (e.g., `ns_list`) to control which namespaces are enabled +- **Solution-based filtering**: Use `current_env.solution_structure` for solution-descriptor–based scenarios + +### Example (cluster type filtering) + +```jinja +{% if current_env.additionalTemplateVariables.env_type == "ocp" %} + - template_path: "{{ templates_dir }}/env_templates/Namespaces/ingress-nginx.yml.j2" + name: ingress-nginx +{% endif %} +``` + +## File Resolution Priority + +If multiple Template Descriptor files exist, EnvGene selects them in descending priority order: + +1. `yml.j2` +2. `yaml.j2` +3. `yml` +4. `yaml` + +Jinja-based descriptors take precedence over static ones. + +## Behavior During Environment Generation + +### Scenario: Generating an Environment with namespace filtering + +1. EnvGene starts Environment Instance generation. +2. EnvGene reads the Template Descriptor (TD). +3. If the TD is a Jinja template (`*.yml.j2` / `*.yaml.j2`), EnvGene renders it first. +4. While rendering, EnvGene evaluates all Jinja `if` conditions for namespaces. +5. EnvGene keeps only the namespaces where the condition is `true`. +6. EnvGene generates the Environment Instance using the final (rendered) TD. + +### What happens when a namespace condition is `false` + +If a namespace is disabled by a condition: + +- EnvGene does **not** create the namespace folder in the Instance Repository +- EnvGene does **not** generate the namespace object +- The namespace does **not** appear in the Effective Set + +> [!NOTE] +> If you are using `NS_BUILD_FILTER`, keep in mind that this parameter only limits which namespaces are processed during a specific pipeline run — it does not add namespaces to the Environment structure. +> +> If a namespace is excluded during Template Descriptor rendering (Jinja condition = `false`), it will not be generated and will not appear in the Instance Repository or Effective Set. +> +> Therefore, such a namespace cannot be processed via `NS_BUILD_FILTER`, because it does not exist in the Environment model. +> For details about `NS_BUILD_FILTER` syntax and usage, see: [Namespace Render Filter](../features/namespace-render-filtering.md) + +## How-To + +For step-by-step instructions, see: +[How to filter namespaces in an Environment Template](/docs/how-to/filter-ns-in-template-descriptor.md) diff --git a/docs/features/resource-profile.md b/docs/features/resource-profile.md index bb2101c20..2ed663e7f 100644 --- a/docs/features/resource-profile.md +++ b/docs/features/resource-profile.md @@ -1,7 +1,7 @@ # Resource Profiles - [Resource Profiles](#resource-profiles) - - [Proposed Approach](#proposed-approach) + - [Overview](#overview) - [Resource Profile Processing During Environment Generation](#resource-profile-processing-during-environment-generation) - [Combination Logic](#combination-logic) - [Naming Rules for Resource Profile Override](#naming-rules-for-resource-profile-override) @@ -9,7 +9,7 @@ - [Merging Logic](#merging-logic) - [Resolving Dot Notation](#resolving-dot-notation) -## Proposed Approach +## Overview Performance deployment parameters like `CPU_LIMIT` and `MEMORY_REQUEST` are grouped separately into Resource Profiles. This makes it manage separately these parameters apart from all other deployment parameters. @@ -17,7 +17,7 @@ The Resource Profiles system has a 3-level hierarchy: 1. Resource Profile Baselines - These are sets of pre-configured performance parameters for services, intended to provide a standardized performance configuration for a service. + These are sets of pre-configured performance parameters for services, intended to provide a standardized performance configuration for a service. Service developers create these baselines and distribute them together with the application artifact. Later, they are included in the Application SBOM. Typical baseline profiles are: @@ -27,7 +27,7 @@ The Resource Profiles system has a 3-level hierarchy: You can have any number of profiles and call them whatever you want, e.g. `small`, `medium`, `large`. -2. [Template Resource Profile Override](/docs/envgene-objects.md#templates-resource-profile-override) +2. [Template Resource Profile Override](/docs/envgene-objects.md#template-resource-profile-override) These are customizations for performance parameters, over a Baseline Resource Profile. Such overrides are created by the configurator in the Template repository, to further adjust performance parameters on top of the Baseline Resource Profile Override for all environments of the same type. @@ -45,10 +45,10 @@ When calculating the [Effective Set](/docs/features/calculator-cli.md#effective- During Environment generation, as part of the [`env_build`](/docs/envgene-pipelines.md#instance-pipeline) job, two types of Resource Profile Overrides are processed and combined: -1. [Template Resource Profile Override](/docs/envgene-objects.md#templates-resource-profile-override) +1. [Template Resource Profile Override](/docs/envgene-objects.md#template-resource-profile-override) - The Template Resource Profile Override is configured individually for each Namespace or Cloud, identified by the `profile.name` property. - Each override is represented as a YAML file located at `/templates/resource_profiles` within the Environment Template repository. + The Template Resource Profile Override is configured individually for each Namespace or Cloud, identified by the `profile.name` property. + Each override is represented as a YAML file located at `/templates/resource_profiles` within the Environment Template repository. The filename (without the `.yaml` or `.yml` extension) must exactly correspond to the value set in `profile.name` for that specific Cloud or Namespace. For example, if a namespace specifies `profile.name: dev-over`, it will use `/templates/resource_profiles/dev-over.yaml` as its template resource profile override. @@ -68,9 +68,9 @@ During Environment generation, as part of the [`env_build`](/docs/envgene-pipeli When an Environment Specific Resource Profile Override is referenced, EnvGene searches for the corresponding YAML file in the Instance repository using the following location priority (from highest to lowest): - 1. `/environments///Inventory/resource_profiles` — Environment-specific, highest priority - 2. `/environments//resource_profiles` — Cluster-wide, applies to all environments in the cluster - 3. `/environments/resource_profiles` — Global, common for the entire repository + 1. `/environments///Inventory/resource_profiles` — Environment-specific, highest priority + 2. `/environments//resource_profiles` — Cluster-wide, applies to all environments in the cluster + 3. `/environments/resource_profiles` — Global, common for the entire repository The first match found is used as the environment-specific override for the given Cloud or Namespace. @@ -78,7 +78,7 @@ The final result of processing is a [Resource Profile Override](/docs/envgene-ob ### Combination Logic -Resource Profile combination happens when you define Environment-Specific Resource Profile Overrides for a particular Cloud or Namespace. +Resource Profile combination happens when you define Environment-Specific Resource Profile Overrides for a particular Cloud or Namespace. These overrides are referenced via `envTemplate.envSpecificResourceProfiles` in the [Environment Inventory](/docs/envgene-configs.md#env_definitionyml): ```yaml @@ -90,14 +90,14 @@ envTemplate: : ``` -There are two ways to combine overrides: **merge** and **replace**. +There are two ways to combine overrides: **merge** and **replace**. Which mode is used is controlled by the `inventory.config.mergeEnvSpecificResourceProfiles` setting in the [Environment Inventory](/docs/envgene-configs.md#env_definitionyml): ```yaml inventory: config: # Optional. Default value - `true` - # If `true`, environment-specific Resource Profile Overrides defined in envTemplate.envSpecificParamsets + # If `true`, environment-specific Resource Profile Overrides defined in envTemplate.envSpecificResourceProfiles # are merged with Resource Profile Overrides from the Environment Template # If `false`, they completely replace the Environment Template's Resource Profile Overrides mergeEnvSpecificResourceProfiles: boolean @@ -121,9 +121,9 @@ The Environment-Specific Resource Profile Override is merged **into** the Templa - If the service exists in both: - For each `parameter` in the template's service, check for a parameter with the same `name` in the target. - If the parameter is missing in the target, add the entire parameter from the template. - - If the parameter exists in both, update the parameter in the target by overwriting its `value` with the one from the template. + - If the parameter exists in both, keep the parameter value from the environment-specific override (env-specific has higher priority). -In this mode, the resulting [Resource Profile Override](/docs/envgene-objects.md#resource-profile-override) will have the same name as the [Template Resource Profile Override](/docs/envgene-objects.md#templates-resource-profile-override). +In this mode, the resulting [Resource Profile Override](/docs/envgene-objects.md#resource-profile-override) will have the same name as the [Template Resource Profile Override](/docs/envgene-objects.md#template-resource-profile-override). ### Naming Rules for Resource Profile Override @@ -140,7 +140,7 @@ inventory: ``` If you set `updateRPOverrideNameWithEnvName: true`, the system will: - + 1. Add a prefix to the name of each [Resource Profile Override](/docs/envgene-objects.md#resource-profile-override). The prefix will be constructed from the [``](/docs/template-macros.md#current_envtenant), [``](/docs/template-macros.md#current_envcloud), and [``](/docs/template-macros.md#current_envname), joined by hyphens, followed by the original Resource Profile Override name. 2. Add the same prefix to the `profile.name` attribute of the Cloud or Namespace diff --git a/docs/features/sbom-retention.md b/docs/features/sbom-retention.md new file mode 100644 index 000000000..b7a439de0 --- /dev/null +++ b/docs/features/sbom-retention.md @@ -0,0 +1,100 @@ +# SBOM Retention + +- [SBOM Retention](#sbom-retention) + - [Overview](#overview) + - [Problem Statement](#problem-statement) + - [Solution](#solution) + - [Retention Strategy](#retention-strategy) + - [Version-Based Strategy](#version-based-strategy) + - [When Cleanup is Triggered](#when-cleanup-is-triggered) + - [Configuration](#configuration) + - [Parameters](#parameters) + - [Examples](#examples) + - [No SBOM cleanup is performed](#no-sbom-cleanup-is-performed) + - [Keep only n most recent versions per application](#keep-only-n-most-recent-versions-per-application) + - [Use Cases](#use-cases) + +## Overview + +SBOM (Software Bill of Materials) files are cached in the Instance Repository to avoid expensive regeneration. This feature provides automatic cleanup of old SBOM files to manage repository size. + +## Problem Statement + +- SBOM generation is a computationally expensive operation +- SBOM files are cached in `/sboms/` directory for reuse +- [Job artifacts](/docs/dev/job-artifacts.md) size limit is 1500 MB +- Without cleanup, the cache grows indefinitely and may reach the size limit + +## Solution + +Automatic SBOM retention policy that: + +- Runs during effective set generation when [GENERATE_EFFECTIVE_SET: true](/docs/instance-pipeline-parameters.md#generate_effective_set) +- Monitors repository size +- Triggers cleanup when size threshold is reached (1200 MB) +- Keeps N most recent versions per application +- Prevents cache growth beyond acceptable limits + +## Retention Strategy + +### Version-Based Strategy + +The version-based strategy keeps the N most recent versions for each application: + +- Each application's SBOMs are stored in `/sboms//`; cleanup processes each such subdirectory +- Sorts versions by file creation time (newest first) +- Keeps the latest N versions per application directory +- Deletes all older versions + +### When Cleanup is Triggered + +Cleanup runs **only** when: + +1. `GENERATE_EFFECTIVE_SET: true` +2. `sbom_retention.enabled: true` in configuration +3. Repository size reaches 1200 MB threshold + +## Configuration + +SBOM retention is configured in `/configuration/config.yml`. + +### Parameters + +```yaml +# Optional +# Triggers only when repository reaches 1200 MB +sbom_retention: + # Optional + # Default value: false + enabled: bool + # Optional + # Default value: 10 + keep_versions_per_app: int +``` + +### Examples + +#### No SBOM cleanup is performed + +```yaml +# No sbom_retention section +``` + +or + +```yaml +sbom_retention: + enabled: false +``` + +#### Keep only n most recent versions per application + +```yaml +sbom_retention: + enabled: true + keep_versions_per_app: n +``` + +## Use Cases + +For detailed step-by-step scenarios demonstrating different SBOM retention configurations and repository states, see [SBOM Retention Use Cases](/docs/use-cases/sbom-retention.md). diff --git a/docs/features/sbom.md b/docs/features/sbom.md index 059321388..c60eff368 100644 --- a/docs/features/sbom.md +++ b/docs/features/sbom.md @@ -8,6 +8,8 @@ - [SBOM types](#sbom-types) - [Application SBOM](#application-sbom) - [Environment Template SBOM](#environment-template-sbom) + - [SBOM Storage and Retention](#sbom-storage-and-retention) + - [SBOM directory layout](#sbom-directory-layout) - [Use Cases](#use-cases) - [Affection map](#affection-map) - [Links](#links) @@ -53,9 +55,38 @@ A JSON file compliant with the CycloneDX specification, describing the following [Example](/examples/env-template.sbom.json) +## SBOM Storage and Retention + +Generated SBOM files are cached in the `/sboms/` directory of the Instance Repository to avoid expensive regeneration. Each application's SBOMs are stored in a subdirectory named after the application. + +To manage repository size and prevent reaching the 1500 MB limit, EnvGene provides automatic SBOM retention. See [SBOM Retention](/docs/features/sbom-retention.md) for configuration details. + +### SBOM directory layout + +SBOM files are stored under `/sboms//`. Each file is named `-.sbom.json`. + +- **Root:** `/sboms/` in the Instance Repository +- **Per application:** one subdirectory `/sboms//` +- **Filename:** `-.sbom.json` + +**Example:** + +```text +... +/sboms +├── Cloud-BSS +| ├── Cloud-BSS-1.2.3.sbom.json +| └── Cloud-BSS-1.2.4.sbom.json +└── cloud-oss + └── cloud-oss-2.0.1.sbom.json +``` + +Full path to a single SBOM: `/sboms//-.sbom.json`. This layout keeps all versions of an application together and simplifies retention cleanup per application. + ## Use Cases -1. TBD +- [SBOM Retention](/docs/use-cases/sbom-retention.md) - Cleanup of old SBOM versions when retention threshold is reached +- [SBOM storage migration](/docs/use-cases/sbom-storage-migration.md) - Automatic migration to per-application layout on first run after upgrading EnvGene ## Affection map diff --git a/docs/features/template-inheritance.md b/docs/features/template-composition.md similarity index 52% rename from docs/features/template-inheritance.md rename to docs/features/template-composition.md index a126557b1..e65b1fe6d 100644 --- a/docs/features/template-inheritance.md +++ b/docs/features/template-composition.md @@ -1,13 +1,16 @@ -# Template Inheritance +# Template Composition -- [Template Inheritance](#template-inheritance) +- [Template Composition](#template-composition) - [Problem Statement](#problem-statement) - [Proposed Approach](#proposed-approach) - [Key Capabilities](#key-capabilities) + - [Detailed Composition Algorithm](#detailed-composition-algorithm) + - [Nested Application and Registry Definitions (appdefs / regdefs)](#nested-application-and-registry-definitions-appdefs--regdefs) + - [Examples (app-reg-defs)](#examples-app-reg-defs) - [Use Cases](#use-cases) - [Case 1](#case-1) - [Case 2](#case-2) - - [Template Inheritance Configuration](#template-inheritance-configuration) + - [Template Composition Configuration](#template-composition-configuration) - [Template Descriptor](#template-descriptor) - [Examples](#examples) - [Child Template Descriptor for One Namespace](#child-template-descriptor-for-one-namespace) @@ -31,7 +34,7 @@ This creates inefficiencies in configuration management, increases error potenti ## Proposed Approach -Introduce **Template Inheritance** - a feature that enables the creation of child templates by inheriting some or all components from one or more parent templates. Supported inheritable components include: +Introduce **Template Composition** - a feature that enables the creation of child templates by composing some or all components from one or more parent templates. Supported components include: - Tenant template - Cloud template @@ -39,17 +42,17 @@ Introduce **Template Inheritance** - a feature that enables the creation of chil This diagram shows parent and child templates with their components. The color of component indicates its source: -![template-inheritance-1.png](/docs/images/template-inheritance-1.png) +![template-composition-1.png](/docs/images/template-composition-1.png) ### Key Capabilities -1. **Selective Inheritance**: - - Inherit some or all components from parents +1. **Selective Composition**: + - Compose some or all components from parents - Optionally override specific parameters of inherited components - Define new components in child template -2. **Component Inheritance Rules**: - - **Inheritable Components**: +2. **Component Composition Rules**: + - **Composable Components**: - Tenant template (cannot be overridden) - Cloud template (override allowed) - Namespace template (override allowed) @@ -62,7 +65,7 @@ This diagram shows parent and child templates with their components. The color o - `e2eParameterSets` - `technicalConfigurationParameterSets` -3. **Inheritance Processing**: +3. **Composition Processing**: - Occurs during child template build in template repository pipeline - Process flow: 1. Download parent template artifacts specified in `parent-templates` section @@ -75,7 +78,110 @@ This diagram shows parent and child templates with their components. The color o 4. **Key Characteristics**: - Built child templates are regular EnvGene artifacts requiring no special handling - Parent templates are regular EnvGene templates needing no special configuration - - Supports multi-level inheritance chains + - Supports multi-level composition chains + +5. **Application & Registry Definitions composition**: + - `appdefs` and `regdefs` live under `templates/` like other resources. Template composition does not merge YAML across files: if the same path appears in more than one source, the file from the later source in copy order replaces the earlier file entirely. + - See [Nested Application and Registry Definitions (appdefs / regdefs)](#nested-application-and-registry-definitions-appdefs--regdefs) below for precedence and how this differs from field-level merge. + +### Detailed Composition Algorithm + +The sequence below describes how composition is executed during template build. + +1. **Discover descriptors** + + Read all `*.yml|*.yaml` files from `templates/env_templates` (top-level only, non-recursive). + Each discovered file is processed as an independent child Template Descriptor. + +2. **Check whether composition is needed** + + If a descriptor does not contain `parent-templates`, composition is skipped for that descriptor. + +3. **Validate parent references** + + - `parent-templates` cannot be empty when declared. + - If multiple parents are declared, each inheriting namespace must explicitly set `parent`. + +4. **Resolve parent artifacts** + + - For each `app:version` in `parent-templates`, find matching Artifact Definition in `configuration/artifact_definitions`. + - Download and unpack the parent template artifact into temporary storage. + +5. **Prepare resource files (`templates/*`)** + + Copy resources in this order: + + - `templates/*` from parents referenced by `namespaces[].parent` + - `templates/*` from `tenant.parent` (if used) + - `templates/*` from `cloud.parent` (if used) + - child `templates/*` (always last) + + > [!IMPORTANT] + > The `templates/*` step is file-based and recursive. It includes all files and directories under `templates/`, including: + > + > - `env_templates` + > - `resource_profiles` + > - `parameters` + > - `appdefs` + > - `regdefs` + > + > If the same relative path exists in multiple sources, the later copy overwrites the earlier one. + +6. **Build resulting Template Descriptor** + + - Create the resulting descriptor as a copy of the child descriptor, then remove `parent-templates` from that resulting descriptor. + - If `tenant: { parent: }` is used, replace child `tenant` with the `tenant` value from the parent template descriptor referenced by ``. + - If `cloud: { parent: }` is used, replace child `cloud` with the `cloud` value from the parent template descriptor referenced by ``. + - If top-level `tenant`, `cloud` or `composite_structure` are missing in child, they are inherited from the first matching parent in namespace iteration order (`first parent wins`) and are not overwritten later. + +7. **Process namespaces** + + - For each namespace with `parent`, find the parent namespace by exact `name` and use it as the base namespace entry in the resulting descriptor. + - Then, if the child namespace defines `template_path`, overwrite the inherited `template_path` with the child value. + +8. **Apply `overrides-parent` (Cloud and Namespace only)** + + - Parameter maps (`deployParameters`, `e2eParameters`, `technicalConfigurationParameters`) are merged into `template_override`. + - Parameter set lists (`deployParameterSets`, `e2eParameterSets`, `technicalConfigurationParameterSets`) are appended into `template_override`. + - `profile` override has two modes: + - default mode (`merge-with-parent` is false or not set): use the override profile as the resulting profile reference; + - merge mode (`merge-with-parent: true`): merge override profile content into the selected parent profile and use the merged result. + - `tenant` does not support `overrides-parent` in template composition (inherit-only behavior). + +9. **Persist output** + + - Save composed descriptor to output `env_templates`. + - Save generated/merged resource profiles to output `resource_profiles`. + +10. **Resulting artifact** + + The output is a regular EnvGene template artifact and does not require special runtime handling. + +> [!IMPORTANT] +> File precedence is copy-order based. If the same file path exists in multiple sources, the last copied file wins. +> Effective precedence is: +> **namespace parents** - **tenant parent** - **cloud parent** - **child template files**. + +### Nested Application and Registry Definitions (appdefs / regdefs) + +Application Definitions (`appdefs`) and Registry Definitions (`regdefs`) can be shipped in parent templates, child templates, or both. Bringing them together during template composition is file-based, not a content merge: overlapping paths are resolved by replacement (last copy wins). + +- **Different relative paths** (for example parent `.../billing-app.yml` and child `.../oss-app.yml`) all remain in the composed artifact; each file is kept as authored. + +- **Same relative path** in more than one source (for example parent and child both ship `templates/appdefs/my-app.yml`): the whole file from the winning source replaces the other. There is **no** field-by-field merge of two YAML documents into one definition. To customize a parent's app or registry file in the child, provide the complete desired file at that path in a source that copies after the parent. + +- Copy order matches template composition precedence: **namespace parents** → **tenant parent** → **cloud parent** → **child template files**. The last source in that order for a given path is what ends up in the built template artifact. + +> [!NOTE] +> After composition, the instance pipeline still runs `app_reg_def_process` on the resulting definition files. +> That pipeline behavior is documented in [app-reg-defs](/docs/features/app-reg-defs.md). +> It does not reintroduce a cross-parent merge for two files that shared the same path. + +#### Examples (app-reg-defs) + +1. Parent ships `templates/appdefs/my-app.yml` and the child adds `templates/appdefs/my-app.yml` with the same relative path. The child's file replaces the parent's. If you only need a small change, duplicate the parent content into the child file and edit the full document there - partial overlays are not applied automatically. + +2. Parent defines one application in `templates/appdefs/app-a.yml` and the child adds `templates/appdefs/app-b.yml`. Both paths are distinct, so both files appear in the composed template. ### Use Cases @@ -85,43 +191,43 @@ This feature can be used in scenarios where EnvGene manages configuration parame A solution comprises multiple applications, where application's teams develop and provide their respective templates. The team responsible for the overall solution collects these templates, combines them into a product-level template, and adds necessary customizations. -![template-inheritance-2.png](/docs/images/template-inheritance-2.png) +![template-composition-2.png](/docs/images/template-composition-2.png) #### Case 2 A solution consists of application groups (domains). Domain teams develop and provide their templates. The product team aggregates these into a product-level template, adding for example integration parameters. Then, a project team customizes this product template for specific project needs. Here, the product template acts as both a parent and child template. -![template-inheritance-3.png](/docs/images/template-inheritance-3.png) +![template-composition-3.png](/docs/images/template-composition-3.png) -### Template Inheritance Configuration +### Template Composition Configuration -Template inheritance is configured in the [Template Descriptor](/docs/envgene-objects.md#template-descriptor) in the child template repository. Below is a description of such a Template Descriptor +Template composition is configured in the [Template Descriptor](/docs/envgene-objects.md#template-descriptor) in the child template repository. Below is a description of such a Template Descriptor #### Template Descriptor ```yaml --- # Optional -# If not set than no Templates Inheritance is assumed +# If not set than no Template Composition is assumed parent-templates: # Key is a parent template name - # Value is a parent template artifact is a in app:ver notation. SNAPSHOT version is not supported + # Value is a parent template artifact in app:ver notation. SNAPSHOT version is not supported default-bss: bss-product-template:2.0.0 basic-template: basic-product-template:10.1.3 # Optional -# If not set, the most recent Composite Structure found in the parent templates referenced by the `namespaces` attribute will be used +# If not set, inherit from the first matching parent encountered during namespace processing (`first parent wins`) composite_structure: "{{ templates_dir }}/env_templates/composite/composite_structure.yml.j2" # Optional -# If not set, the most recent Tenant found in the parent templates referenced by the `namespaces` attribute will be used -# It can be string or dict, if string is provide that means that no inheritance is needed and the exact template will be used +# If not set, inherit from the first matching parent encountered during namespace processing (`first parent wins`) +# It can be string or dict, if string is provide that means that no composition is needed and the exact template will be used # example of string value tenant: "{{ templates_dir }}/env_templates/default/tenant.yml.j2" # example of dict value tenant: parent: basic-template # Optional -# If not set, the most recent Cloud found in the parent templates referenced by the `namespaces` attribute will be used -# It can be string or dict, if string is provide that means that no inheritance is needed and the exact template will be used +# If not set, inherit from the first matching parent encountered during namespace processing (`first parent wins`) +# It can be string or dict, if string is provide that means that no composition is needed and the exact template will be used # example of string value cloud: "{{ templates_dir }}/env_templates/default/cloud.yml.j2" # example of dict value @@ -130,6 +236,9 @@ cloud: # Optional # Section with parameters that should override parent overrides-parent: + # Optional + # Override the name of the cloud in rendering result + name: "override-cloud" # Optional # Section to override resource profile profile: @@ -144,6 +253,7 @@ cloud: baseline-profile-name: dev # Optional. Default value is `false` # Whether to merge parameters from override-profile-name to parent-profile-name + # This mode does not support merging of Jinja template based resource profiles. merge-with-parent: true # Optional # Parameters that extend/override the parent template's values @@ -178,6 +288,9 @@ namespaces: # Optional # Section with parameters that override the parent template's values overrides-parent: + # Optional + # Override the name of the namespace in rendering result + name: "override-ns" # Optional # Section to override resource profile profile: diff --git a/docs/glossary.md b/docs/glossary.md index f6ec07568..ceaa08591 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -1,16 +1,22 @@ # Glossary - [Glossary](#glossary) + - [Custom Params](#custom-params) - [Deploy Postfix](#deploy-postfix) - [Environment](#environment) - [Environment Inventory](#environment-inventory) + - [Environment Template](#environment-template) - [Effective Set](#effective-set) - [Instance Repository](#instance-repository) - [Namespace](#namespace) - - [Template Artifact](#template-artifact) + - [Environment Template Artifact](#environment-template-artifact) This glossary provides definitions of key terms used in the EnvGene documentation. +## Custom Params + +Session-scoped parameters passed via the Instance pipeline parameter [`CUSTOM_PARAMS`](/docs/instance-pipeline-parameters.md#custom_params) and applied to the [Effective Set](/docs/features/calculator-cli.md#version-20-effective-set-structure) with the highest priority. Custom Params are not persisted between parameter calculation sessions and are treated as sensitive. See [Calculator CLI](/docs/features/calculator-cli.md) for details. + ## Deploy Postfix A short identifier for a [Namespace](/docs/envgene-objects.md#namespace) role. Used in the Solution Descriptor. Typically matches the namespace folder name or template name. @@ -23,6 +29,12 @@ A logical grouping representing parameters for deployment target, defined by a u The configuration file describing a specific Environment, including template reference and parameters. See [env_definition.yml](/docs/envgene-configs.md#env_definitionyml). +## Environment Template + +A file structure within the Template Repository describing the structure and parameters of a solution type. Consists of a Template Descriptor and component templates (Tenant, Cloud, Namespaces). Template Descriptors can be Jinja templates (`.yml.j2`, `.yaml.j2`) or static YAML files (`.yml`, `.yaml`). + +A single Environment Template can be used for multiple projects or environment types (commonly referred to as a "unified Environment Template"), with namespace filtering used to include only relevant components for each specific deployment. See [Environment Template Objects](/docs/envgene-objects.md#environment-template-objects) and [Namespace Filtering in Template Descriptor](/docs/features/namespace-filtering-in-template-descriptor.md). + ## Effective Set The complete set of parameters generated for a specific Environment, used by consumers (e.g., ArgoCD). See [Effective Set Structure](/docs/features/calculator-cli.md#version-20-effective-set-structure). @@ -35,6 +47,6 @@ The Git repository containing Environment Inventories, generated Environment Ins An EnvGene object that groups parameters specific to applications within a single namespace in a cluster. Defined in the Environment Instance. See [Namespace](/docs/envgene-objects.md#namespace) -## Template Artifact +## Environment Template Artifact -A versioned template used to generate Environment Instances. Maven artifact. +A versioned Maven artifact containing one or more [Environment Templates](#environment-template) from the Template Repository. Published during Template Repository build process and referenced in the [Environment Inventory](/docs/envgene-configs.md#env_definitionyml) using `application:version` notation (e.g., `project-env-template:v1.2.3`). Used by EnvGene to generate Environment Instances. Also referred to as "Environment Template Artifact". See [Environment Template Objects](/docs/envgene-objects.md#environment-template-objects). diff --git a/docs/how-to/configure-resource-profiles.md b/docs/how-to/configure-resource-profiles.md index 42ab24766..baf3c285c 100644 --- a/docs/how-to/configure-resource-profiles.md +++ b/docs/how-to/configure-resource-profiles.md @@ -16,7 +16,6 @@ - [Use Case 1: Development vs Production Profiles](#use-case-1-development-vs-production-profiles) - [Use Case 2: Cluster-Wide Resource Scaling](#use-case-2-cluster-wide-resource-scaling) - [Use Case 3: Single Environment Hot Fix](#use-case-3-single-environment-hot-fix) - - [Best Practices](#best-practices) - [Verification](#verification) - [Related Documentation](#related-documentation) @@ -93,13 +92,13 @@ Update your Cloud or Namespace template to reference the profile: **Example:** `/templates/namespaces/core.yaml` ```yaml -name: "{{ current_env.name }}-core" -type: namespace +--- +name: "{{ current_env.environmentName }}-core" +# ... other required fields ... profile: name: "dev-core-profile" -applications: - - name: "Cloud-Core" - # ... application configuration ... + baseline: "dev" +# ... other required fields ... ``` ### Step 3: Commit and Publish Template @@ -288,7 +287,7 @@ applications: value: "3000m" ``` -All environments in `prod-cluster-eu` will inherit this profile automatically via **file location priority** (cluster-level overrides apply to all environments within that cluster unless overridden at environment level). +Each environment in `prod-cluster-eu` that references `eu-prod-scaling` via `envTemplate.envSpecificResourceProfiles` in its `env_definition.yml` will use this file. EnvGene finds it automatically via location priority - no need to copy the file per environment, but the reference in `env_definition.yml` is still required. ### Use Case 3: Single Environment Hot Fix @@ -316,37 +315,6 @@ Update `env_definition.yml` for `prod-env-03` only. --- -## Best Practices - -1. **Use Template Profiles for Defaults** - - Define baseline profiles in Template Repository - - Reference appropriate profiles for environment types (dev, staging, prod) - -2. **Use Environment Specific Overrides for Exceptions** - - Only override when environment needs differ from template defaults - - Document why specific values are needed - -3. **Organize by Scope** - - - **Environment-specific**: High-traffic environments, special requirements - - **Cluster-wide**: Regional or infrastructure-based differences - - **Global**: Cross-cluster standards - -4. **Use Descriptive Names** - - Include environment type, purpose, or special notes - - Examples: `prod-high-traffic`, `eu-cluster-baseline`, `hotfix-temp-scaling` - -5. **Document Decisions** - - Add `description` field explaining why values were chosen - - Link to performance test results or incidents if applicable - -6. **Version Control Best Practices** - - Keep resource profiles in version control - - Review changes carefully (resource changes affect costs and stability) - - Test in lower environments first - ---- - ## Verification After configuring resource profiles, verify they are applied correctly: @@ -360,7 +328,7 @@ After configuring resource profiles, verify they are applied correctly: Look for the Resource Profile Override in the generated output: ```text - Namespaces//resource_profiles/.yml + environments///Profiles/.yml ``` 3. **Verify Merge Result:** diff --git a/docs/how-to/create-cluster.md b/docs/how-to/create-cluster.md index 50e1d9b87..8979177a4 100644 --- a/docs/how-to/create-cluster.md +++ b/docs/how-to/create-cluster.md @@ -86,8 +86,8 @@ In this approach, the [Cloud](/docs/envgene-objects.md) is generated from the [C - Collect all required parameters necessary to define the [Cloud](/docs/envgene-objects.md). - Assemble the [Cloud Passport](/docs/envgene-objects.md#cloud-passport) using the expected format. Refer to the sample: - - [cluster-01.yml](/docs/samples/instance-repository/environmentscluster-01/cloud-passport/cluster-01.yml) - - [cluster-01-creds.yml](/docs/samples/instance-repository/environmentscluster-01/cloud-passport/cluster-01-creds.yml) + - [cluster-01.yml](/docs/samples/instance-repository/environments/cluster-01/cloud-passport/cluster-01.yml) + - [cluster-01-creds.yml](/docs/samples/instance-repository/environments/cluster-01/cloud-passport/cluster-01-creds.yml) - Place it under the right location: `/environments//cloud-passport/` Example: @@ -121,7 +121,7 @@ In this approach, the [Cloud](/docs/envgene-objects.md#cloud) is generated using 1. The Instance and Discovery repositories has already been initialized and follows the required structure. 2. Integration with Discovery repository is configured in the Instance repository: - + `/configuration/integration.yml`: ```yaml diff --git a/docs/how-to/defining-complex-parameters-using-yaml-objects.md b/docs/how-to/defining-complex-parameters-using-yaml-objects.md new file mode 100644 index 000000000..dbdc8fed3 --- /dev/null +++ b/docs/how-to/defining-complex-parameters-using-yaml-objects.md @@ -0,0 +1,324 @@ +# Defining and Managing Complex Parameters in EnvGene Using YAML Objects + +- [Defining and Managing Complex Parameters in EnvGene Using YAML Objects](#defining-and-managing-complex-parameters-in-envgene-using-yaml-objects) + - [Overview](#overview) + - [When to Use This Guide](#when-to-use-this-guide) + - [Core Rule](#core-rule) + - [Why YAML Objects Are Required](#why-yaml-objects-are-required) + - [Structural Integrity](#structural-integrity) + - [Clean and Meaningful Git Diffs](#clean-and-meaningful-git-diffs) + - [No Manual Escaping](#no-manual-escaping) + - [How to Define Complex Parameters Correctly](#how-to-define-complex-parameters-correctly) + - [Defining a Map (Object)](#defining-a-map-object) + - [Defining a List](#defining-a-list) + - [How EnvGene Processes Complex Parameters](#how-envgene-processes-complex-parameters) + - [During Environment Instance Generation](#during-environment-instance-generation) + - [During CMDB Import](#during-cmdb-import) + - [End-to-End Example](#end-to-end-example) + - [YAML Definition in Template or Environment-specific parameters](#yaml-definition-in-template-or-environment-specific-parameters) + - [Effective Set Output](#effective-set-output) + - [CMDB Imported Representation](#cmdb-imported-representation) + - [Design Principles for YAML](#design-principles-for-yaml) + - [Treat YAML as Structured Data](#treat-yaml-as-structured-data) + - [Preserve Type Integrity](#preserve-type-integrity) + - [Avoid Embedded JSON Inside YAML](#avoid-embedded-json-inside-yaml) + - [Migration Strategy for Existing Multiline or JSON Strings](#migration-strategy-for-existing-multiline-or-json-strings) + - [Operational Impact](#operational-impact) + - [Final Recommendation](#final-recommendation) + +## Overview + +This guide explains how to define complex parameters (maps and lists) in EnvGene using native YAML objects instead of multiline or JSON strings. It is intended for engineers working with Git-managed EnvGene instance or template repositories. + +### When to Use This Guide + +Use this guide when: + +- Defining new parameters in EnvGene +- Refactoring existing multiline string parameters +- Debugging CMDB import formatting issues +- Reviewing pull requests involving complex configuration +- Establishing configuration standards across teams + +### Core Rule + +> Always define complex parameters as structured YAML objects. Never use multiline strings or JSON strings for structured configuration. + +Complex parameters are maps (objects) and lists (arrays). Maps can contain other maps or lists as values. + +## Why YAML Objects Are Required + +### Structural Integrity + +When defined as YAML objects: + +- YAML schema validation works +- Linters and IDE tooling function correctly + +When defined as multiline strings: + +- No validation is applied +- Type safety is broken + +### Clean and Meaningful Git Diffs + +Structured YAML enables semantic diffs; only the changed keys appear in the diff: + +```diff + deploymentConfig: + replicas: 2 + strategy: + type: RollingUpdate + maxUnavailable: 0 + resources: + limits: +- memory: 512Mi ++ memory: 1Gi + cpu: 500m + requests: + memory: 256Mi + cpu: 250m +``` + +Escaped JSON in a string shows the entire value as one long line; a single field change (e.g. `memory`) is buried in escaped quotes and hard to review: + +```diff +- deploymentConfig: "{ \"replicas\":2,\"strategy\":{\"type\":\"RollingUpdate\",\"maxUnavailable\":0},\"resources\":{\"limits\":{\"memory\":\"512Mi\",\"cpu\":\"500m\"},\"requests\":{\"memory\":\"256Mi\",\"cpu\":\"250m\"}} }" ++ deploymentConfig: "{ \"replicas\":2,\"strategy\":{\"type\":\"RollingUpdate\",\"maxUnavailable\":0},\"resources\":{\"limits\":{\"memory\":\"1Gi\",\"cpu\":\"500m\"},\"requests\":{\"memory\":\"256Mi\",\"cpu\":\"250m\"}} }" +``` + +Structured YAML improves: + +- Code reviews +- Drift detection +- Merge conflict resolution + +### No Manual Escaping + +Multiline or escaped JSON string (e.g. a small map): + +```yaml +config: "{ \"limits\": { \"cpu\": \"500m\" } }" +``` + +Native YAML: + +```yaml +config: + limits: + cpu: 500m +``` + +Manual escaping: + +- Is error-prone +- Breaks readability +- Causes CMDB import issues + +## How to Define Complex Parameters Correctly + +### Defining a Map (Object) + +Correct: + +```yaml +parameters: + resourceLimits: + cpu: 500m + memory: 1Gi +``` + +Incorrect (multiline string - map is stored as text): + +```yaml +parameters: + resourceLimits: | + cpu: 500m + memory: 1Gi +``` + +Incorrect (JSON string): + +```yaml +parameters: + resourceLimits: '{"cpu":"500m","memory":"1Gi"}' +``` + +### Defining a List + +Correct: + +```yaml +parameters: + allowedIPs: + - 10.10.0.1 + - 10.10.0.2 + - 10.10.0.3 +``` + +Incorrect (multiline string - list is stored as text): + +```yaml +parameters: + allowedIPs: | + - 10.10.0.1 + - 10.10.0.2 +``` + +Incorrect (JSON string): + +```yaml +parameters: + allowedIPs: '["10.10.0.1","10.10.0.2","10.10.0.3"]' +``` + +## How EnvGene Processes Complex Parameters + +### During Environment Instance Generation + +EnvGene preserves structure and types (map, list, boolean, number, string) and produces a structured effective set. No flattening or string conversion occurs at this stage. + +Example effective set output: + +```yaml +deploymentConfig: + replicas: 3 + strategy: + type: RollingUpdate + maxUnavailable: 1 + resources: + limits: + cpu: 1 + memory: 2Gi +``` + +### During CMDB Import + +CMDB requires complex parameters to be stored as escaped string representations. + +EnvGene automatically transforms the YAML object. + +Source YAML object: + +```yaml +deploymentConfig: + replicas: 3 + strategy: + type: RollingUpdate +``` + +CMDB stores the parameter key separately; the value is the escaped JSON string. Example value: + +```json +"{\"replicas\":3,\"strategy\":{\"type\":\"RollingUpdate\"}}" +``` + +Key points: + +- Conversion is automatic +- No manual escaping required +- Original structure drives transformation +- Type fidelity is preserved before serialization + +## End-to-End Example + +### YAML Definition in Template or Environment-specific parameters + +```yaml +parameters: + serviceConfig: + service: + name: payment-api + port: 8080 + monitoring: + enabled: true + endpoints: + - /health + - /metrics +``` + +### Effective Set Output + +```yaml +serviceConfig: + service: + name: payment-api + port: 8080 + monitoring: + enabled: true + endpoints: + - /health + - /metrics +``` + +### CMDB Imported Representation + +```json +"{\"service\":{\"name\":\"payment-api\",\"port\":8080},\"monitoring\":{\"enabled\":true,\"endpoints\":[\"/health\",\"/metrics\"]}}" +``` + +Result: + +- Structure preserved +- Types preserved +- Converted into CMDB-compatible format automatically + +## Design Principles for YAML + +### Treat YAML as Structured Data + +If the parameter represents structured configuration, it must be a YAML object, not a multiline string or a JSON string. + +### Preserve Type Integrity + +Avoid: + +```yaml +replicas: "3" +enabled: "true" +``` + +Prefer: + +```yaml +replicas: 3 +enabled: true +``` + +### Avoid Embedded JSON Inside YAML + +Do not embed JSON in YAML values (e.g. `config: '{"limits":{"cpu":"1"}}'`). See [No Manual Escaping](#no-manual-escaping). + +## Migration Strategy for Existing Multiline or JSON Strings + +If complex parameters already exist as multiline strings or JSON in a string: + +1. Identify parameters to migrate: + - **Multiline:** look for keys whose value uses `|` or `>` (block scalars) with YAML-like or list-like content underneath. + - **JSON in string:** look for values that are single-quoted or double-quoted JSON (e.g. `'{"key":...}'` or `"{ \"key\": ... }"`). +2. Convert content into structured YAML (native map or list). +3. Validate: + - Instance generation + - Effective set output + - CMDB import result +4. Remove manual escaping where it was used for JSON. +5. Add YAML linting to CI pipelines. + +## Operational Impact + +Adopting YAML objects improves: + +- Maintainability +- Git readability +- Configuration correctness +- CI/CD validation capability +- Reduction of CMDB import failures +- Long-term configuration scalability + +## Final Recommendation + +- Use native YAML objects only for maps and lists (see [Core Rule](#core-rule)). +- Rely on automatic CMDB transformation and enforce YAML linting in CI/CD. +- Keep configuration declarative and type-safe. + +YAML objects enable that; multiline and JSON strings undermine it. diff --git a/docs/how-to/docker-registry-configuration.md b/docs/how-to/docker-registry-configuration.md new file mode 100644 index 000000000..f090457f8 --- /dev/null +++ b/docs/how-to/docker-registry-configuration.md @@ -0,0 +1,168 @@ +# Using Docker Registries in EnvGene GitHub Workflow + +- [Using Docker Registries in EnvGene GitHub Workflow](#using-docker-registries-in-envgene-github-workflow) + - [Description](#description) + - [Supported Registries](#supported-registries) + - [Prerequisites](#prerequisites) + - [How the Workflow Uses the Registry](#how-the-workflow-uses-the-registry) + - [Option 1: GitHub Container Registry (GHCR)](#option-1-github-container-registry-ghcr) + - [GHCR Configuration](#ghcr-configuration) + - [GHCR Authentication](#ghcr-authentication) + - [Option 2: Google Artifact Registry (GAR)](#option-2-google-artifact-registry-gar) + - [GAR Prerequisites](#gar-prerequisites) + - [Step 1: Configure GitHub Repository Variables](#step-1-configure-github-repository-variables) + - [Step 2: Add GCP\_SA\_KEY Secret](#step-2-add-gcp_sa_key-secret) + - [GAR Authentication Flow](#gar-authentication-flow) + - [Parameter Reference](#parameter-reference) + - [Switching Between Registries](#switching-between-registries) + - [Troubleshooting](#troubleshooting) + +## Description + +This guide explains how to configure the EnvGene GitHub workflow to pull Docker images from different registries. The workflow uses three EnvGene images: `qubership-envgene`, `qubership-pipegene`, and `qubership-effective-set-generator`. By default, these images are pulled from GitHub Container Registry (GHCR). You can switch to Google Artifact Registry (GAR) by configuring the appropriate variables and secrets. + +## Supported Registries + +The EnvGene GitHub workflow currently supports two Docker registries: + +- **GitHub Container Registry (GHCR)** - Default option, no additional configuration required +- **Google Artifact Registry (GAR)** - Requires GCP service account configuration + +> [!NOTE] +> For other registry types (AWS ECR, Azure ACR, custom registries), custom authentication steps need to be added to the workflow. + +## Prerequisites + +- Instance repository with the EnvGene GitHub workflow installed +- Access to **Settings → Secrets and variables → Actions** in your GitHub repository + +## How the Workflow Uses the Registry + +The workflow defines image names in the `env` section of `Envgene.yml`: + +```yaml +env: + DOCKER_IMAGE_NAME_ENVGENE: "${{ vars.DOCKER_REGISTRY || 'ghcr.io/netcracker' }}/qubership-envgene" + DOCKER_IMAGE_NAME_PIPEGENE: "${{ vars.DOCKER_REGISTRY || 'ghcr.io/netcracker' }}/qubership-pipegene" + DOCKER_IMAGE_NAME_EFFECTIVE_SET_GENERATOR: "${{ vars.DOCKER_REGISTRY || 'ghcr.io/netcracker' }}/qubership-effective-set-generator" +``` + +The `DOCKER_REGISTRY` variable (from repository variables) determines the registry base. When `DOCKER_CLOUD_REGISTRY_PROVIDER` is set to `GCP`, the workflow runs an authentication step before pulling images from GAR. + +## Option 1: GitHub Container Registry (GHCR) + +GHCR is the default registry. No additional configuration is required if your images are hosted at `ghcr.io/netcracker`. + +### GHCR Configuration + +| Where to configure | Parameter | Value | +|--------------------------|-------------------|----------------------| +| **Settings → Variables** | `DOCKER_REGISTRY` | `ghcr.io/netcracker` | + +If you omit `DOCKER_REGISTRY`, the workflow uses `ghcr.io/netcracker` by default. + +### GHCR Authentication + +GitHub Actions automatically authenticates to `ghcr.io` using `GITHUB_TOKEN` when pulling images. No extra secrets are needed. Ensure your repository has access to the container images (e.g. via package permissions if the images are in a different organization). + +## Option 2: Google Artifact Registry (GAR) + +To use Google Artifact Registry, you must configure the registry URL, set the cloud provider, and provide a GCP service account key for authentication. + +### GAR Prerequisites + +- Registry URL (path to your GAR repository) +- GCP service account JSON key with access to the registry + +### Step 1: Configure GitHub Repository Variables + +1. Go to your repository on GitHub. +2. Open **Settings → Secrets and variables → Actions**. +3. Open the **Variables** tab. +4. Add or edit the following variables: + +| Variable | Value | +|----------------------------------|----------------------------------------------| +| `DOCKER_REGISTRY` | `REGION-docker.pkg.dev/PROJECT_ID/REPO_NAME` | +| `DOCKER_CLOUD_REGISTRY_PROVIDER` | `GCP` | + +**Example for `DOCKER_REGISTRY`:** + +```text +europe-west1-docker.pkg.dev/my-gcp-project/envgene-images +``` + +Replace: + +- `REGION` with your GAR region (e.g. `europe-west1`, `us-central1`) +- `PROJECT_ID` with your GCP project ID +- `REPO_NAME` with your Artifact Registry repository name + +### Step 2: Add GCP_SA_KEY Secret + +1. In **Settings → Secrets and variables → Actions**, open the **Secrets** tab. +2. Click **New repository secret**. +3. Name: `GCP_SA_KEY`. +4. Value: Paste the full JSON content of the GCP service account key (including `{` and `}`). +5. Click **Add secret**. + +> [!IMPORTANT] +> The secret must contain the full JSON. Do not truncate or modify it. The workflow uses it with `docker login -u _json_key --password-stdin`. + +### GAR Authentication Flow + +When `DOCKER_CLOUD_REGISTRY_PROVIDER` is set to `GCP`, the workflow runs this step in the `envgene_execution` job: + +```yaml +- name: Authenticate to GAR (Google Artifact Registry) + if: needs.process_environment_variables.outputs.DOCKER_CLOUD_REGISTRY_PROVIDER == 'GCP' + run: | + REGISTRY_HOST=$(echo "${{ vars.DOCKER_REGISTRY }}" | cut -d'/' -f1) + echo '${{ secrets.GCP_SA_KEY }}' | docker login -u _json_key --password-stdin "$REGISTRY_HOST" +``` + +The step extracts the registry host (e.g. `europe-west1-docker.pkg.dev`) from `DOCKER_REGISTRY` and authenticates before any Docker image pulls. + +## Parameter Reference + +| Parameter | Location | Required for GHCR | Required for GAR | +|----------------------------------|-----------|-----------------------|------------------------| +| `DOCKER_REGISTRY` | Variables | No (uses default) | Yes | +| `DOCKER_CLOUD_REGISTRY_PROVIDER` | Variables | No | Yes - set to `GCP` | +| `GCP_SA_KEY` | Secrets | No | Yes | + +> [!NOTE] +> For GHCR: If you omit `DOCKER_REGISTRY`, the workflow uses the default value `ghcr.io/netcracker`. Set `DOCKER_REGISTRY` only if your images are in a different GHCR organization or path. + +## Switching Between Registries + +To switch from GHCR to GAR: + +1. Add `DOCKER_REGISTRY` and `DOCKER_CLOUD_REGISTRY_PROVIDER` variables. +2. Add `GCP_SA_KEY` secret. +3. Trigger the workflow. The GAR authentication step will run automatically. + +To switch back to GHCR: + +1. Remove or clear `DOCKER_CLOUD_REGISTRY_PROVIDER` (or set it to empty). +2. Set `DOCKER_REGISTRY` to `ghcr.io/netcracker` (or remove it to use the default). +3. Optionally remove `GCP_SA_KEY` if no longer needed. + +## Troubleshooting + +**Authentication fails with "unauthorized" or "denied":** + +- Verify `GCP_SA_KEY` contains the full JSON key. +- Ensure the service account has `Artifact Registry Reader` role on the repository. +- Check that the key has not expired. + +**Image pull fails with "not found":** + +- Verify `DOCKER_REGISTRY` format: `REGION-docker.pkg.dev/PROJECT_ID/REPO_NAME`. +- Ensure the images exist in the repository with the expected names: `qubership-envgene`, `qubership-pipegene`, `qubership-effective-set-generator`. +- Check the image tags match those in the workflow (e.g. `1.31.9`). + +**GAR authentication step does not run:** + +- Confirm `DOCKER_CLOUD_REGISTRY_PROVIDER` is set to `GCP` (case-sensitive). +- Variables are passed via `process_environment_variables` job outputs. Ensure the variable is not overridden in `pipeline_vars.env` with an empty value. diff --git a/docs/how-to/envgene-maitanance.md b/docs/how-to/envgene-maitanance.md index a0e7b5d86..6e05faee2 100644 --- a/docs/how-to/envgene-maitanance.md +++ b/docs/how-to/envgene-maitanance.md @@ -52,7 +52,8 @@ Run the GSF package manager on your local machine with the following command: git-system-follower install \ -r \ -b \ - -t + -t \ + --extra self_token no-masked ``` **Parameter Details:** @@ -60,7 +61,10 @@ git-system-follower install \ - ``: Docker image path from Step 1 - ``: Project instance repository URL (format: `https://git.com/project.git`) - ``: Branch of project instance repository -- ``: Project instance repository token from Initial Setup Step 2 +- ``: Project instance repository token from Initial Setup Step 2 (used both for GSF Git operations and for EnvGene to commit changes) + +> [!NOTE] +> The same token is used twice: `-t` for GSF to authenticate git operations, and `--extra self_token` to configure EnvGene's repository access. **Example:** @@ -69,5 +73,6 @@ git-system-follower install \ docker.io/envgene/instance:1.2.3 \ -r https://git.qubership.org/configuration-management/env-instance.git \ -b master \ - -t token-placeholder-123 + -t token-placeholder-123 \ + --extra self_token token-placeholder-123 no-masked ``` diff --git a/docs/how-to/extend-github-instance-pipeline.md b/docs/how-to/extend-github-instance-pipeline.md new file mode 100644 index 000000000..9dcaa3ce8 --- /dev/null +++ b/docs/how-to/extend-github-instance-pipeline.md @@ -0,0 +1,441 @@ +# Extension Pipeline & apply_envgene_patch Documentation + +This document describes the GitLab CI extension pipeline and the `apply_envgene_patch` script used to customize the EnvGene GitHub Actions workflow for your instance. + +In the [qubership-envgene](https://github.com/Netcracker/qubership-envgene) repository, the Python scripts (`apply_envgene_patch.py`, `git_commit.py`) live under **`github_workflows/instance-repo-pipeline/extend_logic/scripts/`**. The `qubership-instance-repo-pipeline` Docker image copies that folder into the container at **`/opt/github/extend_logic/scripts/`**. + +--- + +## Table of Contents + +1. [Overview](#overview) +2. [Pipeline Flow](#pipeline-flow) +3. [GitLab CI configuration](#gitlab-ci-configuration) +4. [apply_envgene_patch Script](#apply_envgene_patch-script) +5. [Patch File Format](#patch-file-format) +6. [Operations Reference](#operations-reference) +7. [Examples](#examples) + +--- + +## Overview + +The pipeline uses the Docker image `qubership-instance-repo-pipeline` from [Netcracker/qubership-envgene](https://github.com/Netcracker/qubership-envgene) and extends the base EnvGene workflow by adding new variables and components to `Envgene.yml` - as **steps** or **jobs** depending on the project. + +**Flow:** + +1. **Init & apply** — `apply_envgene_patch.py` copies the base workflow from `/opt/github` into a **staging directory** `extended_github_instance_pipeline//`, applies YAML patch files (components), then by default **packs the result into** **`extended_github_instance_pipeline/.zip`** and deletes the staging folder. `` comes from `DOCKER_IMAGE_TAG` or `INSTANCE_REPO_PIPELINE_IMAGE_TAG` (default `latest`). If you pass **no patch files**, only the base snapshot is written as a ZIP (nothing merged or inserted). Use **`--output-format dir`** to keep the versioned folder instead of a ZIP. +2. **Commit & push** — `git_commit.py` commits and pushes changes under `extended_github_instance_pipeline/` (ZIP files and, if used, leftover directories) + +This allows instance repositories to extend the base EnvGene workflow with custom variables, steps, and configuration without forking the entire workflow. + +--- + +## Pipeline Flow + +The pipeline is defined in `.gitlab-ci.yml` in your instance repository and runs in the `qubership-instance-repo-pipeline` Docker image (from [qubership-envgene](https://github.com/Netcracker/qubership-envgene)). Scripts are executed from paths inside the image: `/opt/github/extend_logic/scripts/` (same files as in the repository under `github_workflows/instance-repo-pipeline/extend_logic/scripts/`). + +A complete copypaste example for `.gitlab-ci.yml` is in [GitLab CI configuration](#gitlab-ci-configuration). + +**Steps:** + +| Step | Description | +|------|-------------| +| Init & apply | Runs `apply_envgene_patch.py`: removes `extended_github_instance_pipeline//` (staging) and `.github/`, copies `/opt/github`, applies patches, then writes **`extended_github_instance_pipeline/.zip`** by default and removes the staging dir | +| Commit & push | Commits changes and pushes to the current branch | + +**Requirements:** + +- `GITLAB_TOKEN` with `write_repository` scope (for `git_commit.py`) +- `DOCKER_IMAGE_TAG` or `INSTANCE_REPO_PIPELINE_IMAGE_TAG` set so the snapshot path matches the pipeline image tag (see [GitLab CI configuration](#gitlab-ci-configuration)) +- Pipeline runs on schedule or manual trigger (not on push/MR by default) + +--- + +## GitLab CI configuration + +Copy the following into the root of your instance repository as `.gitlab-ci.yml`, or merge the `extend-the-gh-pipeline` job into an existing file. The image must be a build of [instance-repo-pipeline](https://github.com/Netcracker/qubership-envgene/tree/main/github_workflows/instance-repo-pipeline) whose Dockerfile copies **`extend_logic/scripts/`** from that directory into **`/opt/github/extend_logic/scripts/`** in the image. + +**Placeholders:** + +| Placeholder | Description | +|-------------|-------------| +| `DOCKER_IMAGE_NAME` | Container image name without tag (for example `registry.example.com/org/qubership-instance-repo-pipeline`) | +| `DOCKER_IMAGE_TAG` | Image tag (for example `1.2.3` or `latest`). Must match the pipeline image tag; the artifact file is **`extended_github_instance_pipeline/.zip`**. | +| `PATH_TO_COMPONENT` | Zero or more patch YAML files in your repository (for example `components/component-a.yaml`). Omit all arguments to only materialize the base workflow as a ZIP (no merges). | + +GitLab artifacts should still publish **`extended_github_instance_pipeline/`** as a whole. By default each run produces **`extended_github_instance_pipeline/.zip`** (for example `extended_github_instance_pipeline/1.2.3.zip`). Extract the ZIP to get `workflows/`, `configuration/`, and so on at the archive root. + +```yaml +--- +#Variables +variables: + INSTANCE_REPO_PIPELINE_IMAGE: DOCKER_IMAGE_NAME + INSTANCE_REPO_PIPELINE_IMAGE_TAG: DOCKER_IMAGE_TAG + DOCKER_IMAGE_TAG: "${INSTANCE_REPO_PIPELINE_IMAGE_TAG}" + +#Rules +workflow: + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event"' + when: never + - when: always + +#Stages +stages: + - extend-pipeline + +extend-the-gh-pipeline: + stage: extend-pipeline + image: + name: ${INSTANCE_REPO_PIPELINE_IMAGE}:${INSTANCE_REPO_PIPELINE_IMAGE_TAG} + script: + - python3 /opt/github/extend_logic/scripts/apply_envgene_patch.py PATH_TO_COMPONENT + - python3 /opt/github/extend_logic/scripts/git_commit.py + artifacts: + paths: + - extended_github_instance_pipeline/ +``` + +To run **without component patches** (only pack the base workflow into **`extended_github_instance_pipeline/.zip`**), use a script line with no patch arguments: + +```yaml + - python3 /opt/github/extend_logic/scripts/apply_envgene_patch.py +``` + +To **keep a directory** instead of a ZIP (for example for debugging), add **`--output-format dir`**: + +```yaml + - python3 /opt/github/extend_logic/scripts/apply_envgene_patch.py --output-format dir PATH_TO_COMPONENT +``` + +--- + +## apply_envgene_patch Script + +**Location:** `git_commit.py` is in the same directory as `apply_envgene_patch.py`. + +- **qubership-envgene tree:** `github_workflows/instance-repo-pipeline/extend_logic/scripts/` +- **`qubership-instance-repo-pipeline` image:** `/opt/github/extend_logic/scripts/` + +**Usage:** + +```bash +# From the root of a qubership-envgene clone - CI / full run (init from /opt/github + apply patches) +python3 github_workflows/instance-repo-pipeline/extend_logic/scripts/apply_envgene_patch.py components/component-a.yaml components/variables.yaml + +# Local run (skip init, apply patches to existing output dir) +python3 github_workflows/instance-repo-pipeline/extend_logic/scripts/apply_envgene_patch.py --no-init components/component-a.yaml components/variables.yaml + +# Custom output dir +python3 github_workflows/instance-repo-pipeline/extend_logic/scripts/apply_envgene_patch.py --output-dir my_pipeline components/component-a.yaml + +# Keep a versioned directory instead of a ZIP (no packaging step) +python3 github_workflows/instance-repo-pipeline/extend_logic/scripts/apply_envgene_patch.py --output-format dir components/component-a.yaml +``` + +Adjust the path to the script if your working directory is not the repository root. + +**Environment variables (versioned name):** + +| Variable | Description | +|----------|-------------| +| `DOCKER_IMAGE_TAG` | Preferred. Used in **`extended_github_instance_pipeline/.zip`** (or folder `/` with `--output-format dir`). | +| `INSTANCE_REPO_PIPELINE_IMAGE_TAG` | Used if `DOCKER_IMAGE_TAG` is unset (same value as the pipeline image tag in CI). | +| (none) | Defaults to `latest`. | + +Patches are applied against the **staging tree** **`//`**. Unless **`--output-format dir`** is used, that directory is removed after **`extended_github_instance_pipeline/.zip`** is written. + +**Options:** + +| Option | Description | +|--------|-------------| +| `--output-dir DIR` | Parent directory for artifacts and staging. Maps `target_file` paths starting with `.github/` into the staging directory. Default: `extended_github_instance_pipeline`. | +| `--output-format` | **ZIP** (default): after patching, create **`/.zip`** and delete staging. **Directory** (`dir`): keep **`//`** and do not create a ZIP file. | +| `--init-from DIR` | Before applying patches: remove `//` and `.github`, copy DIR into the staging directory. Default: `/opt/github`. | +| `--no-init` | Skip init step. Use when the staging directory already exists (e.g. local runs without `/opt/github`). | + +**Default behavior:** The script initializes **`//`**, applies patches (if any), then **writes a ZIP archive** unless **`--output-format dir`**. If you pass **no patch file paths**, it only initializes and writes the base snapshot as a ZIP. Use `--no-init` for local runs when the staging tree already exists. + +**Dependencies:** `ruamel.yaml` (install via `pip install ruamel.yaml`) + +The script reads patch files and applies a sequence of operations to target files. Each operation has an `action` and optional `target_file` (defaults to the first operation's target). Paths like `.github/workflows/Envgene.yml` are resolved to **`//workflows/Envgene.yml`** during processing (that path exists only until the ZIP packaging step when using the default format). + +--- + +## Patch File Format + +Patch files are YAML documents containing a list of operations. Use `target_file` paths starting with `.github/` — they are resolved under the **staging** snapshot root (e.g. `.github/workflows/Envgene.yml` → `extended_github_instance_pipeline//workflows/Envgene.yml` before the result is packaged as a ZIP): + +```yaml +--- +- target_file: .github/workflows/Envgene.yml + action: merge + section: DOCKER_IMAGE_NAMES + content: + DOCKER_IMAGE_NAME_DEPLOYTOOL: "my-registry/my-image" + +- target_file: .github/pipeline_vars.env + action: merge + content: + MY_VAR: "value" +``` + +--- + +## Operations Reference + +### 1. Merge (action: `merge`) + +Adds or updates key-value pairs in the target file. + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `target_file` | Yes* | Path to target file | +| `action` | Yes | `merge` | +| `content` | Yes | Dict of key-value pairs | +| `section` | For YAML | Section comment (e.g. `#DOCKER_IMAGE_NAMES`) | +| `path` | For YAML | Dotted path (e.g. `jobs.process_environment_variables.outputs`) | + +**Merge variants:** + +- **.env files** — Merges `KEY=value`; no section/path needed +- **YAML with section** — Merges into block after `#SECTION` comment +- **YAML with path** — Merges into block at dotted path (e.g. `jobs.job_name.outputs`) + +--- + +### 2. Insert (action: `insert`) + +Inserts a block of content at a specific position. Requires exactly one anchor. + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `target_file` | Yes* | Path to target file | +| `action` | Yes | `insert` | +| `content` | Yes | String or YAML block to insert | +| `after_section` | One of | Insert after `### SECTION - END ###` | +| `before_section` | One of | Insert before `### SECTION - START ###` | +| `after_step` | One of | Insert after step with given name | +| `before_step` | One of | Insert before step with given name | +| `skip_if_present` | No | Skip if this string is already in the file | + +**Formatting:** Inserted content gets 2 empty lines before and after; indentation is preserved. + +--- + +## Examples + +### Example 1: Merge Docker image names (section-based) + +Add or override Docker image names in the `#DOCKER_IMAGE_NAMES` section: + +```yaml +--- +- path: env + target_file: .github/workflows/Envgene.yml + action: merge + section: DOCKER_IMAGE_NAMES + content: + DOCKER_IMAGE_NAME_DEPLOYTOOL: "ghcr.io/myorg/deploytool" + DOCKER_IMAGE_NAME_ENVGENE: "ghcr.io/myorg/envgene" +``` + +--- + +### Example 2: Merge job outputs (path-based) + +Add outputs to a specific job: + +```yaml +--- +- path: jobs.process_environment_variables.outputs + target_file: .github/workflows/Envgene.yml + action: merge + content: + MY_CUSTOM_OUTPUT: "custom_value" + ANOTHER_OUTPUT: ${{ env.SOME_VAR }} +``` + +--- + +### Example 3: Merge .env variables + +Add or update variables in `.github/pipeline_vars.env`: + +```yaml +--- +- target_file: .github/pipeline_vars.env + action: merge + content: + CONFIG_VAR_1: "value1" + CONFIG_VAR_2: "value2" + PIPELINE_VAR: "my_value" +``` + +--- + +### Example 4: Insert block after section (with markers) + +Insert a new step block after `### GIT COMMIT - END ###`: + +```yaml +--- +- target_file: .github/workflows/Envgene.yml + action: insert + after_section: "GIT COMMIT" + content: | + ### MY CUSTOM STEP - START ### + - name: My Custom Step + run: echo "Hello from custom step" + ### MY CUSTOM STEP - END ### +``` + +--- + +### Example 5: Insert block before section + +Insert content before a section starts: + +```yaml +--- +- target_file: .github/workflows/Envgene.yml + action: insert + before_section: "BG MANAGE" + content: | + - name: Pre-BG step + run: echo "Running before BG MANAGE" +``` + +--- + +### Example 6: Insert by step name (no section markers) + +Insert a step after "Prepare environment" or before "Create env file for container": + +```yaml +--- +- target_file: .github/workflows/Envgene.yml + action: insert + after_step: "Prepare environment" + content: | + - name: Create name for dynamic secret + run: | + SECRET_NAME=$(echo "${{ matrix.environment }}" | awk -F "/" '{print $1}')_${{ env.SECRET_POSTFIX }} + echo "SECRET_NAME=$SECRET_NAME" >> $GITHUB_ENV +``` + +Or insert before a step: + +```yaml +--- +- target_file: .github/workflows/Envgene.yml + action: insert + before_step: "Create env file for container" + content: | + - name: My step before Create env file + run: echo "pre-step" +``` + +**Step name matching:** Case-insensitive, supports partial match (e.g. `"Prepare environment"` matches `"Prepare environment"`). + +--- + +### Example 7: Skip if already present + +Avoid duplicate insertion: + +```yaml +--- +- target_file: .github/workflows/Envgene.yml + action: insert + after_section: "GIT COMMIT" + skip_if_present: "### MY CUSTOM STEP - START ###" + content: | + ### MY CUSTOM STEP - START ### + - name: My Custom Step + ... +``` + +--- + +### Example 8: Full component file (component-a.yaml) + +```yaml +--- +- path: env + target_file: .github/workflows/Envgene.yml + action: merge + section: DOCKER_IMAGE_NAMES + content: + DOCKER_IMAGE_NAME_DEPLOYTOOL: "MY_VALUE" + +- path: env + target_file: .github/workflows/Envgene.yml + action: merge + section: DOCKER_IMAGE_TAGS + content: + DOCKER_IMAGE_TAG_DEPLOYTOOL: "MY_VALUE" + +- path: jobs.process_environment_variables.outputs + target_file: .github/workflows/Envgene.yml + action: merge + content: + MY_NEW_VARIABLE: "MY_VALUE" + +- path: jobs.envgene_execution.steps + target_file: .github/workflows/Envgene.yml + action: insert + after_section: "GIT COMMIT" + content: | + ### MY CUSTOM STEP - START ### + - name: My Custom Step + if: needs.process_environment_variables.outputs.MY_FEATURE_ENABLED == 'true' + env: + DYNAMIC_SECRET: ${{ secrets[env.SECRET_NAME] }} + run: | + echo "Custom step logic..." + ### MY CUSTOM STEP - END ### +``` + +--- + +## Envgene.yml Structure (Reference) + +The base workflow uses these extension points: + +| Extension point | Type | Example | +|-----------------|------|---------| +| `#DOCKER_IMAGE_NAMES` | Section comment | Merge env vars | +| `#DOCKER_IMAGE_TAGS` | Section comment | Merge env vars | +| `jobs.process_environment_variables.outputs` | Dotted path | Merge job outputs | +| `### SECTION - START ###` / `### SECTION - END ###` | Section markers | Insert steps | +| Step names | `- name: "..."` | Insert by step anchor | + +--- + +## Adding New Patch Files + +1. Create a YAML file in `components/` (e.g. `components/my-feature.yaml`) +1. Add operations following the format above +1. Register the file in `.gitlab-ci.yml` (paths below match scripts inside the pipeline image): + +```yaml +- python3 /opt/github/extend_logic/scripts/apply_envgene_patch.py components/component-a.yaml components/variables.yaml components/my-feature.yaml +``` + +1. Patches are applied in order; later patches can override or extend earlier ones. + +--- + +## Troubleshooting + +| Error | Cause | Solution | +|-------|-------|----------| +| `Source directory not found: /opt/github` | Running locally without Docker image | Use `--no-init` and ensure output-dir exists, or `--init-from` a local path | +| `Section #X not found` | Section comment missing in target | Add `#SECTION` comment or use `path` | +| `Marker '### X - END ###' not found` | Section markers missing | Add `### SECTION - START ###` and `### SECTION - END ###` | +| `Step 'X' not found` | Step name doesn't match | Use exact or partial step name (case-insensitive) | +| `Block 'path' not found` | Dotted path invalid | Verify YAML structure (jobs → job_name → outputs) | +| `File not found` | Wrong target path or output-dir missing | Use path relative to repository root; run with init or `--no-init` on existing dir | +| `Invalid DOCKER_IMAGE_TAG` | Tag contains path separators | Use a plain tag only (for example `1.2.3`), not a path | diff --git a/docs/how-to/filter-ns-in-template-descriptor.md b/docs/how-to/filter-ns-in-template-descriptor.md new file mode 100644 index 000000000..d898262b0 --- /dev/null +++ b/docs/how-to/filter-ns-in-template-descriptor.md @@ -0,0 +1,160 @@ +# How to filter namespaces in an Environment Template + +- [How to filter namespaces in an Environment Template](#how-to-filter-namespaces-in-an-environment-template) + - [Overview](#overview) + - [Step 1. Convert Template Descriptor to a Jinja Template](#step-1-convert-template-descriptor-to-a-jinja-template) + - [Step 2. Wrap namespaces in Jinja conditions](#step-2-wrap-namespaces-in-jinja-conditions) + - [Generic pattern](#generic-pattern) + - [Example](#example) + - [Step 3. Create the namespace list file](#step-3-create-the-namespace-list-file) + - [Example ns-list.yaml](#example-ns-listyaml) + - [Step 4. Reference the file in env\_definition.yml](#step-4-reference-the-file-in-env_definitionyml) + - [Step 5. Run Environment Instance generation](#step-5-run-environment-instance-generation) + - [Step 6. Verify the result](#step-6-verify-the-result) + - [See also](#see-also) + +## Overview + +This guide explains how to generate an Environment that includes only a selected subset of namespaces from a unified Environment Template. It follows the feature described in [Namespace Filtering in Template Descriptor](/docs/features/namespace-filtering-in-template-descriptor.md). + +Use this approach if: + +- You use a unified Environment Template with multiple namespaces +- You need to include or exclude namespaces depending on specific environment need + +--- + +## Step 1. Convert Template Descriptor to a Jinja Template + +Find the Template Descriptor file: + +`/templates/env_templates/.yaml` + +Rename it to one of the following: + +- `.yaml.j2` +or +- `.yml.j2` + +After renaming, EnvGene will treat the Template Descriptor as a Jinja template and render it before Environment generation. + +--- + +## Step 2. Wrap namespaces in Jinja conditions + +Each namespace can be conditionally included using a Jinja `if` expression. + +### Generic pattern + +```jinja +namespaces: +{% if current_env.additionalTemplateVariables.ns_list.get('namespace-key', false) %} + - template_path: {{ templates_dir }}/env_templates/Namespaces/.yml.j2 + deploy_postfix: +{% endif %} +``` + +**Parameters**: + +- namespace-key - key in `ns_list` used to enable or disable the namespace +- template_path - path to the Namespace Template file +- deploy_postfix - folder name in Instance Repository (`/Namespaces//`) + +### Example + +```jinja +namespaces: +{% if current_env.additionalTemplateVariables.ns_list.get('postgresql', false) %} + - template_path: {{ templates_dir }}/env_templates/Namespaces/postgresql.yml.j2 + deploy_postfix: postgresql +{% endif %} + +{% if current_env.additionalTemplateVariables.ns_list.get('postgresql-dbaas', false) %} + - template_path: {{ templates_dir }}/env_templates/Namespaces/postgresql-dbaas.yml.j2 + deploy_postfix: postgresql-dbaas +{% endif %} + +{% if current_env.additionalTemplateVariables.ns_list.get('kafka', false) %} + - template_path: {{ templates_dir }}/env_templates/Namespaces/kafka.yml.j2 + deploy_postfix: kafka +{% endif %} + +{% if current_env.additionalTemplateVariables.ns_list.get('platform-monitoring', false) %} + - template_path: {{ templates_dir }}/env_templates/Namespaces/platform-monitoring.yml.j2 + deploy_postfix: platform-monitoring +{% endif %} +``` + +If the key is missing or set to false, the namespace will not be generated. + +--- + +## Step 3. Create the namespace list file + +Create a YAML file that defines which namespaces are enabled. The file must be in a location where EnvGene resolves [shared template variable files](/docs/envgene-configs.md#env_definitionyml) (for example, under `/environments/` or per-cluster/per-environment, depending on your Instance Repository layout). The filename (without extension) is what you will reference in `env_definition.yml` in Step 4. + +Example: if the file is named `ns-list.yaml`, use the name `ns-list` in `sharedTemplateVariables`. + +```yaml +# Example path: /environments/shared-template-variables/ns-list.yaml +# The top-level key (ns_list) becomes available as current_env.additionalTemplateVariables.ns_list + +ns_list: + : + : +``` + +### Example ns-list.yaml + +```yaml +# /environments/shared-template-variables/ns-list.yaml + +ns_list: + platform-monitoring: true + postgresql: true + postgresql-dbaas: true + kafka: false +``` + +Set `true` for namespaces that must be generated. + +## Step 4. Reference the file in env_definition.yml + +In the [Environment Inventory](/docs/envgene-configs.md#env_definitionyml) for the environment, add the shared template variable by **filename without extension** (e.g. `ns-list` for `ns-list.yaml`): + +```yaml +envTemplate: + sharedTemplateVariables: + - ns-list +``` + +The content of the file is merged into `additionalTemplateVariables`, so `ns_list` from the YAML is available during Template Descriptor rendering as `current_env.additionalTemplateVariables.ns_list`. + +## Step 5. Run Environment Instance generation + +Run the Environment Instance generation process using the standard EnvGene pipeline. + +During generation: + +- The Template Descriptor is rendered +- Jinja conditions are evaluated +- Only enabled namespaces are included + +If a condition evaluates to `false`: + +- The namespace folder is not created +- The namespace object is not generated + +## Step 6. Verify the result + +After generation, verify the directory: `/environments///Namespaces/` + +Only namespaces that passed the Jinja conditions should be present. + +--- + +## See also + +- [Namespace Filtering in Template Descriptor](/docs/features/namespace-filtering-in-template-descriptor.md) - Feature overview, file resolution priority, and behavior when a condition is `false` +- [env_definition.yml](/docs/envgene-configs.md#env_definitionyml) - Environment Inventory and `sharedTemplateVariables` +- [Template Descriptor](/docs/envgene-objects.md#template-descriptor) - Location and Jinja extensions diff --git a/docs/how-to/generate-effective-set.md b/docs/how-to/generate-effective-set.md new file mode 100644 index 000000000..8e642030c --- /dev/null +++ b/docs/how-to/generate-effective-set.md @@ -0,0 +1,212 @@ +# How to Generate an Effective Set + +- [How to Generate an Effective Set](#how-to-generate-an-effective-set) + - [Description](#description) + - [Prerequisites](#prerequisites) + - [Steps](#steps) + - [1. Verify the Inventory Is Configured](#1-verify-the-inventory-is-configured) + - [2. Prepare the Solution Descriptor](#2-prepare-the-solution-descriptor) + - [3. Trigger the Pipeline](#3-trigger-the-pipeline) + - [Results](#results) + - [Other Common Scenarios](#other-common-scenarios) + - [Generate Without a Solution Descriptor](#generate-without-a-solution-descriptor) + - [Inject High-Priority Parameters at Runtime](#inject-high-priority-parameters-at-runtime) + +## Description + +This guide explains how to trigger a full Effective Set generation for a specific environment in the Instance Repository. + +The **Effective Set** is the final resolved parameter set for an environment - the output consumed by ArgoCD and other deployment tools. EnvGene calculates it by merging parameters from multiple sources in strict priority order. + +The Effective Set is written to `environments///effective-set/`. + +> [!IMPORTANT] +> The pipeline run is **not atomic**. If any earlier job fails, the Effective Set is not updated. You do not need to prepare the Environment Instance manually - the pipeline builds it as part of the same run. + +## Prerequisites + +1. An Instance Repository exists with the target environment's Inventory configured under `environments///Inventory/` +2. A template artifact is published and accessible (e.g. `env-template:2.5.0`) +3. A Solution Descriptor artifact is published and accessible (e.g. `sd:1.2.3`) - or you intend to generate only the `topology` and `pipeline` contexts (see [Generate Without a Solution Descriptor](#generate-without-a-solution-descriptor)) + +--- + +## Steps + +### 1. Verify the Inventory Is Configured + +Confirm that the environment's Inventory is present in the repository: + +```text +environments/prod-cluster/prod-01/ +└── Inventory/ + └── env_definition.yml +``` + +If `env_definition.yml` is missing, create it before proceeding. See [Environment Inventory](/docs/envgene-configs.md#env_definitionyml). + +--- + +### 2. Prepare the Solution Descriptor + +Provide the SD via one of the following pipeline variables. The pipeline writes it to `environments/prod-cluster/prod-01/Inventory/solution-descriptor/sd.yaml` automatically. + +**Option A - Artifact reference (SD_SOURCE_TYPE + SD_VERSION):** + +The SD is fetched from an artifact registry at pipeline start and written to the repository path above: + +```text +SD_SOURCE_TYPE: artifact +SD_VERSION: sd:1.2.3 +``` + +**Option B - Inline content (SD_SOURCE_TYPE + SD_DATA):** + +Pass the SD content as a JSON string directly as a pipeline variable. Useful for testing or one-off runs: + +```text +SD_SOURCE_TYPE: json +SD_DATA: '{"applications":[{"version":"Cloud-BSS:1.2.3","deployPostfix":"bss"},{"version":"cloud-oss:2.0.1","deployPostfix":"oss"}]}' +``` + +If neither is provided, the existing `sd.yaml` file already committed in the repository is used. + +--- + +### 3. Trigger the Pipeline + +Trigger the Instance pipeline with the following variables: + +| Variable | Value | Description | +|--------------------------|------------------------|----------------------------------------------------| +| `ENV_NAMES` | `prod-cluster/prod-01` | Target environment; comma-separated for multiple | +| `ENV_BUILDER` | `true` | Rebuild the Environment Instance from the template | +| `ENV_TEMPLATE_VERSION` | `env-template:2.5.0` | Template version to use for the Instance build | +| `GENERATE_EFFECTIVE_SET` | `true` | Enable the effective set generation job | +| `SD_SOURCE_TYPE` | `artifact` | How the SD is provided (`artifact` or `json`) | +| `SD_VERSION` | `sd:1.2.3` | SD artifact name and version | + +The pipeline executes the following job sequence: + +```text +appregdef_render → process_sd → env_build → generate_effective_set → git_commit +``` + +If only the Effective Set needs to be regenerated without rebuilding the Environment Instance, set `ENV_BUILDER: false`. The `generate_effective_set` job will use the existing Instance files from the previous run. + +> [!IMPORTANT] +> The `generate_effective_set` job always depends on `env_build`. If `ENV_BUILDER: false` is set but the Environment Instance files are already present from a previous run, generation proceeds normally. + +--- + +## Results + +A successful run produces the following structure under `environments/prod-cluster/prod-01/effective-set/`: + +```text +effective-set/ +├── topology/ +│ ├── parameters.yaml +│ └── credentials.yaml +├── pipeline/ +│ ├── parameters.yaml +│ └── credentials.yaml +├── deployment/ +│ ├── mapping.yml +│ ├── bss/ +│ │ └── Cloud-BSS/ +│ │ └── values/ +│ │ ├── deployment-parameters.yaml +│ │ ├── collision-deployment-parameters.yaml +│ │ ├── credentials.yaml +│ │ ├── collision-credentials.yaml +│ │ ├── deploy-descriptor.yaml +│ │ ├── custom-params.yaml +│ │ └── per-service-parameters/ +│ │ └── cloud-bss-7b/ +│ │ └── deployment-parameters.yaml +│ └── oss/ +│ └── cloud-oss/ +│ └── values/ +│ ├── deployment-parameters.yaml +│ ├── collision-deployment-parameters.yaml +│ ├── credentials.yaml +│ ├── collision-credentials.yaml +│ ├── deploy-descriptor.yaml +│ ├── custom-params.yaml +│ └── per-service-parameters/ +│ └── cloud-oss/ +│ └── deployment-parameters.yaml +├── runtime/ +│ ├── mapping.yml +│ ├── bss/ +│ │ └── Cloud-BSS/ +│ │ ├── parameters.yaml +│ │ └── credentials.yaml +│ └── oss/ +│ └── cloud-oss/ +│ ├── parameters.yaml +│ └── credentials.yaml +└── cleanup/ + ├── mapping.yml + ├── bss/ + │ ├── parameters.yaml + │ └── credentials.yaml + └── oss/ + ├── parameters.yaml + └── credentials.yaml +``` + +The `git_commit` job commits these files to the Instance Repository automatically. The pipeline run is complete when `git_commit` succeeds and the files appear under `environments/prod-cluster/prod-01/effective-set/`. + +--- + +## Other Common Scenarios + +### Generate Without a Solution Descriptor + +When no Solution Descriptor is provided - for example when setting up infrastructure-only namespaces or preparing an environment before any applications are defined - the Effective Set is generated in a partial mode. + +Without an SD, EnvGene does not know which applications belong to which namespaces, so the application-specific contexts cannot be produced. The following contexts are generated normally: + +- `topology` - cluster structure, namespace mapping, composite structure, BG domain +- `pipeline` - orchestration pipeline parameters + +The following contexts are **not generated**: + +- `deployment` - requires application definitions from the SD +- `runtime` - requires application definitions from the SD +- `cleanup` - requires application definitions from the SD + +To use this mode, simply omit `SD_VERSION` and `SD_SOURCE_TYPE` from the pipeline variables. If no SD artifact is passed and no `sd.yaml` exists in the repository, EnvGene skips all application-level processing automatically. + +--- + +### Inject High-Priority Parameters at Runtime + +Use `CUSTOM_PARAMS` to inject parameters at the highest priority level, overriding everything else. This is useful for temporary overrides, incident response, or injecting session-specific values. + +`CUSTOM_PARAMS` accepts a JSON-in-string value: + +```json +{ + "deployment": { + "FEATURE_FLAG_NEW_BILLING": "true", + "MAX_RETRIES": "5" + }, + "runtime": { + "LOG_LEVEL": "DEBUG" + } +} +``` + +Set this as a pipeline variable: + +```text +CUSTOM_PARAMS: '{"deployment":{"FEATURE_FLAG_NEW_BILLING":"true","MAX_RETRIES":"5"}}' +``` + +The injected parameters are written to `custom-params.yaml` inside each application's `values/` folder, applied at the highest priority level after all other values files. + +> [!WARNING] +> Custom Params override all other parameter sources. Use them only for temporary, session-specific values. Do not use Custom Params to replace permanent configuration - use Environment Specific ParameterSets instead. diff --git a/docs/how-to/sbom-storage-migration.md b/docs/how-to/sbom-storage-migration.md new file mode 100644 index 000000000..3ace9b072 --- /dev/null +++ b/docs/how-to/sbom-storage-migration.md @@ -0,0 +1,39 @@ +# Migrate SBOM Storage to Per-Application Layout + +This guide describes how migration to the per-application SBOM layout works when you run EnvGene with a version that uses the new storage structure. + +- [Migrate SBOM Storage to Per-Application Layout](#migrate-sbom-storage-to-per-application-layout) + - [When to use this guide](#when-to-use-this-guide) + - [How migration works](#how-migration-works) + - [What to expect](#what-to-expect) + +## When to use this guide + +Use this guide when: + +- Your Instance Repository currently has SBOM files stored directly under `/sboms/` (flat layout) +- You are upgrading EnvGene (Instance pipeline) to a version that uses the new per-application SBOM layout + +After the first run with the new version, [SBOM Retention](/docs/features/sbom-retention.md) and Effective Set generation will use the new paths. See [SBOM directory layout](/docs/features/sbom.md#sbom-directory-layout) for the target structure. + +## How migration works + +Migration is **automatic** when you run the Instance pipeline with an EnvGene version that implements the new SBOM storage structure: + +1. **Old SBOMs are removed** - SBOM files in the previous flat location (`/sboms/*.sbom.json`) are deleted by EnvGene. +2. **New SBOMs are generated** - SBOMs are generated and written to the new layout: `/sboms//-.sbom.json`. + +You do **not** need to move or copy files manually. The first pipeline run that includes the `generate_effective_set` job (e.g. with `GENERATE_EFFECTIVE_SET: true`) and uses the new EnvGene version will clear the old flat SBOMs and produce SBOMs in the per-application directories. Because SBOM generation can be expensive, that run may take longer than usual while SBOMs are recreated. + +Example: + +- **Before:** `/sboms/Cloud-BSS-1.2.3.sbom.json`, `/sboms/cloud-oss-2.0.1.sbom.json` +- **After:** `/sboms/Cloud-BSS/Cloud-BSS-1.2.3.sbom.json`, `/sboms/cloud-oss/cloud-oss-2.0.1.sbom.json` + +For the formal scenario (pre-requisites, trigger, steps, results), see [SBOM Storage Migration Use Case](/docs/use-cases/sbom-storage-migration.md). + +## What to expect + +- **First run with new EnvGene version:** Flat SBOMs under `/sboms/` are removed; SBOMs are generated into `/sboms//`. The job may take longer while SBOMs are regenerated. +- **Later runs:** Only the new layout is used; no migration step runs again. +- **Rollback:** If you revert to an older EnvGene version that expects the flat layout, you would need to run that version again; it may regenerate SBOMs in the old flat structure (behavior depends on that version). diff --git a/docs/images/template-inheritance-1.png b/docs/images/template-composition-1.png similarity index 100% rename from docs/images/template-inheritance-1.png rename to docs/images/template-composition-1.png diff --git a/docs/images/template-inheritance-2.png b/docs/images/template-composition-2.png similarity index 100% rename from docs/images/template-inheritance-2.png rename to docs/images/template-composition-2.png diff --git a/docs/images/template-inheritance-3.png b/docs/images/template-composition-3.png similarity index 100% rename from docs/images/template-inheritance-3.png rename to docs/images/template-composition-3.png diff --git a/docs/instance-pipeline-parameters.md b/docs/instance-pipeline-parameters.md index 308f57f41..82285064e 100644 --- a/docs/instance-pipeline-parameters.md +++ b/docs/instance-pipeline-parameters.md @@ -14,6 +14,7 @@ - [`ENV_INVENTORY_CONTENT`](#env_inventory_content) - [`GENERATE_EFFECTIVE_SET`](#generate_effective_set) - [`EFFECTIVE_SET_CONFIG`](#effective_set_config) + - [`CUSTOM_PARAMS`](#custom_params) - [`APP_REG_DEFS_JOB`](#app_reg_defs_job) - [`APP_DEFS_PATH`](#app_defs_path) - [`REG_DEFS_PATH`](#reg_defs_path) @@ -32,6 +33,7 @@ - [Deprecated Parameters](#deprecated-parameters) - [`SD_DELTA`](#sd_delta) - [Archived Parameters](#archived-parameters) + - [Multiple Values Support](#multiple-values-support) The following are the launch parameters for the instance repository pipeline. These parameters influence, the execution of specific jobs within the pipeline. @@ -41,7 +43,10 @@ All parameters are of the string data type ### `ENV_NAMES` -**Description**: Specifies the environment(s) for which processing will be triggered. Uses the `/` notation. If multiple environments are provided, they must be separated by a `\n` (newline) delimiter. In multi-environment case, each environment will trigger its own independent pipeline flow. All environments will use the same set of pipeline parameters (as documented in this spec) +**Description**: Specifies the environment(s) for which processing will be triggered. Uses the `/` notation. + +If specifying more than one environment, separate them as described in [Multiple Values Support](#multiple-values-support). +For multiple environments, each environment will initiate its own independent pipeline flow, using the same set of pipeline parameters for all. **Default Value**: None @@ -50,7 +55,11 @@ All parameters are of the string data type **Example**: - Single environment: `ocp-01/platform` -- Multiple environments (separated by \n) `k8s-01/env-1\nk8s-01/env2` +- Multiple environments: + - `k8s-01/env-1\nk8s-01/env2` + - `k8s-01/env-1;k8s-01/env2` + - `k8s-01/env-1,k8s-01/env2` + - `k8s-01/env-1 k8s-01/env2` ### `ENV_BUILDER` @@ -120,7 +129,7 @@ This parameter serves as a configuration for an extension point. Integration wit **Allowed values**: - `PERSISTENT` (default) - Applies the standard behavior: the pipeline updates the template version in Environment Inventory by updating `envTemplate.artifact` (or `envTemplate.templateArtifact.artifact.version`) in `env_definition.yml`. + Applies the standard behavior: the pipeline updates the template version in Environment Inventory by modifying `envTemplate.artifact` (or `envTemplate.templateArtifact.artifact.version`) in `env_definition.yml`. - `TEMPORARY` Applies `ENV_TEMPLATE_VERSION` **only for the current pipeline execution** and **does not** update `envTemplate.artifact` (or `envTemplate.templateArtifact.artifact.version`) in `env_definition.yml`. @@ -281,6 +290,40 @@ Consumer-specific pipeline context components registered in EnvGene: "{\"version\": \"v2.0\", \"app_chart_validation\": \"false\"}" ``` +### `CUSTOM_PARAMS` + +**Description**: Session-scoped parameters injected into the Effective Set during parameter calculation. Custom Params are not persisted across parameter calculation sessions, have the highest priority in the parameter resolution hierarchy, and are treated as sensitive. + +`CUSTOM_PARAMS` is only applied when [`GENERATE_EFFECTIVE_SET`](#generate_effective_set) is `true`. If `GENERATE_EFFECTIVE_SET` is `false`, the `generate_effective_set` job does not run and `CUSTOM_PARAMS` has no effect. + +EnvGene passes the value unchanged to the Calculator CLI via `--custom-params`. See [Calculator CLI](/docs/features/calculator-cli.md) for how Custom Params are applied to the Effective Set. + +**Format**: A string containing a JSON object (JSON-in-string). The JSON object must conform to the [schema](/schemas/custom-params.schema.json). + +```json +{ + "deployment": { + "": "", + "...": "..." + }, + "runtime": { + "": "", + "...": "..." + } +} +``` + +> [!NOTE] +> +> 1. `` can be complex, i.e. a map or a list +> 2. All keys are optional + +**Default Value**: None + +**Mandatory**: No + +**Example**: `"{\"deployment\":{\"MY_OVERRIDE\":\"value\"}}"` + ### `APP_REG_DEFS_JOB` **Description**: Specifies the name of the job that is the source of [Application Definition](/docs/envgene-objects.md#application-definition) and [Registry Definitions](/docs/envgene-objects.md#registry-definition). @@ -341,7 +384,9 @@ See details in [SD processing](/docs/features/sd-processing.md) ### `SD_VERSION` -**Description**: Specifies one or more SD artifacts in `application:version` notation passed via a `\n` separator. +**Description**: Specifies one or more SD artifacts in `application:version` notation. + +If specifying more than one environment, separate them as described in [Multiple Values Support](#multiple-values-support). EnvGene downloads and sequentially merges them in the `basic-merge` mode, where subsequent `application:version` takes priority over the previous one. Optionally saves the result to [Delta SD](/docs/features/sd-processing.md#delta-sd), then merges with [Full SD](/docs/features/sd-processing.md#full-sd) using `SD_REPO_MERGE_MODE` merge mode @@ -354,16 +399,22 @@ See details in [SD processing](/docs/features/sd-processing.md) **Example**: - Single SD: `MONITORING:0.64.1` -- Multiple SD (separated by \n) `solution-part-1:0.64.2\nsolution-part-2:0.44.1` +- Multiple SDs: + - `solution-part-1:0.64.2\nsolution-part-2:0.44.1` + - `solution-part-1:0.64.2;solution-part-2:0.44.1` + - `solution-part-1:0.64.2,solution-part-2:0.44.1` + - `solution-part-1:0.64.2 solution-part-2:0.44.1` ### `SD_DATA` -**Description**: Specifies the contents of one or more SD in JSON-in-string format. Can be either a single SD object or a list of SD objects. +**Description**: Specifies the contents of one or more SD. Can be either a single SD object or a list of SD objects. If a single SD object is provided, it is processed directly. If a list is provided, EnvGene sequentially merges them in the `basic-merge` mode, where subsequent element takes priority over the previous one. Optionally saves the result to [Delta SD](/docs/features/sd-processing.md#delta-sd), then merges with [Full SD](/docs/features/sd-processing.md#full-sd) using `SD_REPO_MERGE_MODE` merge mode See details in [SD processing](/docs/features/sd-processing.md) +**Format**: A string containing a JSON object (JSON-in-string) + **Default Value**: None **Mandatory**: No @@ -423,7 +474,7 @@ See details in [Namespace Render Filtering](/docs/features/namespace-render-filt **Description**: Operation identifier in Envgene. Must be a valid [UUID v4](https://www.rfc-editor.org/rfc/rfc4122). This parameter is used in two scenarios: -1. If this parameter is provided, the resulting pipeline commit will include a [Git trailer](https://git-scm.com/docs/git-commit#Documentation/git-commit.txt-code--trailerlttokengtltvaluegtcode) in the format: `DEPLOYMENT_SESSION_ID: `. +1. If this parameter is provided, the resulting pipeline commit will include a [Git trailer](https://git-scm.com/docs/git-commit#Documentation/git-commit.txt---trailertokenvalue) in the format: `DEPLOYMENT_SESSION_ID: `. 2. It will also be part of the deployment context of the Effective Set. The EnvGene passes it to the Calculator command-line tool using the `--extra_params` attribute. In this case it is used together with `GENERATE_EFFECTIVE_SET`. **Default Value**: None @@ -450,13 +501,13 @@ See details in [Namespace Render Filtering](/docs/features/namespace-render-filt } ``` -| Attribute | Mandatory | Description | Default | Example | -|-------------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|---------| -| `namespace` | Mandatory | The name of the Namespace where the parameter to be modified is defined | None | `env-1-platform-monitoring` | -| `application` | Optional | The name of the Application (sub-resource under `namespace`) where the parameter to be modified is defined. Cannot be used with `pipeline` context | None | `MONITORING` | -| `context` | Mandatory | The context of the parameter being modified. Valid values: `pipeline`, `deployment`, `runtime` | None | `deployment` | -| `parameter_key` | Mandatory | The name (key) of the parameter to be modified | None | `login` or `db.connection.password` | -| `parameter_value` | Mandatory | New value (plaintext or encrypted). Envgene, depending on the value of the [`crypt`](/docs/envgene-configs.md#configyml) attribute, will either decrypt, encrypt, or leave the value unchanged. If an encrypted value is passed, it must be encrypted with a key that Envgene can decrypt. | None | `admin` | +| Attribute | Mandatory | Description | Default | Example | +|-------------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|-------------------------------------| +| `namespace` | Mandatory | The name of the Namespace where the parameter to be modified is defined | None | `env-1-platform-monitoring` | +| `application` | Optional | The name of the Application (sub-resource under `namespace`) where the parameter to be modified is defined. Cannot be used with `pipeline` context | None | `MONITORING` | +| `context` | Mandatory | The context of the parameter being modified. Valid values: `pipeline`, `deployment`, `runtime` | None | `deployment` | +| `parameter_key` | Mandatory | The name (key) of the parameter to be modified | None | `login` or `db.connection.password` | +| `parameter_value` | Mandatory | New value (plaintext or encrypted). Envgene, depending on the value of the [`crypt`](/docs/envgene-configs.md#configyml) attribute, will either decrypt, encrypt, or leave the value unchanged. If an encrypted value is passed, it must be encrypted with a key that Envgene can decrypt | None | `admin` | **Default Value**: None @@ -619,3 +670,29 @@ See details in [SD processing](/docs/features/sd-processing.md) ## Archived Parameters These parameters are no longer in use and are maintained for historical reference + +## Multiple Values Support + +Some pipeline parameters support multiple values. +Values can be separated using one of the following delimiters: + +- Newline (`\n`) +- Semicolon (`;`) +- Comma (`,`) +- Space (` `) + +**Example:** + +```text +# Using newline +k8s-01/env-1\nk8s-01/env-2 + +# Using comma +k8s-01/env-1,k8s-01/env-2 + +# Using semicolon +k8s-01/env-1;k8s-01/env-2 + +# Using space +k8s-01/env-1 k8s-01/env-2 +``` diff --git a/docs/template-macros.md b/docs/template-macros.md index da5cf8601..76f2392d5 100644 --- a/docs/template-macros.md +++ b/docs/template-macros.md @@ -70,7 +70,8 @@ - [`cloud`](#cloud) - [`deployer`](#deployer) - [Deprecated Credential Macros](#deprecated-credential-macros) - - [`${envgene.creds.get('').username|password|secret}`](#envgenecredsgetcred-idusernamepasswordsecret) + - [`${envgen.creds.get('').username|password|secret}`](#envgencredsgetcred-idusernamepasswordsecret) + - [`${cmdb.creds.get('').username|password|secret}`](#cmdbcredsgetcred-idusernamepasswordsecret) - [Deprecated Calculator CLI macros](#deprecated-calculator-cli-macros) - [`BASELINE_PROJ`](#baseline_proj) @@ -1209,7 +1210,7 @@ ${creds.get('').username|password|secret} Where `username`, `password`, and `secret` are **credential fields** that define the type of sensitive data being referenced. -For each `` during Environment Instance generation a [Credential](/docs/envgene-objects.md#credential) object is created in the [Environment Credential File](/docs/envgene-objects.md#environment-credential-file) +For each `` during Environment Instance generation a [Credential](/docs/envgene-objects.md#credential) object is created in the [Environment Credential File](/docs/envgene-objects.md#environment-credentials-file) Type assignment: @@ -1224,7 +1225,7 @@ kafka_password: ${creds.get('kafka-cred').password} k8s_token: ${creds.get('k8s-cred').secret} ``` -**Usage in sample:** [Sample](/docs/samples/templates/parameters/migration/test-deploy-creds.yml) +**Usage in sample:** [Sample](/docs/samples/template-repository/templates/parameters/migration/test-deploy-creds.yml) ## Deprecated Macros @@ -1256,9 +1257,11 @@ k8s_token: ${creds.get('k8s-cred').secret} ### Deprecated Credential Macros -#### `${envgene.creds.get('').username|password|secret}` +#### `${envgen.creds.get('').username|password|secret}` -**Description:** This macro was used for processing system sensitive parameters—parameters that EnvGene uses to integrate itself with external systems, such as the login and password for a registry or a token for a GitLab instance. +**Replacement**: [`${creds.get('').username|password|secret}`](#credential-macro) + +#### `${cmdb.creds.get('').username|password|secret}` **Replacement**: [`${creds.get('').username|password|secret}`](#credential-macro) diff --git a/docs/test-cases/sd-processing.md b/docs/test-cases/sd-processing.md new file mode 100644 index 000000000..69b437830 --- /dev/null +++ b/docs/test-cases/sd-processing.md @@ -0,0 +1 @@ +# SD Processing Test Cases diff --git a/docs/tutorials/effective-set.md b/docs/tutorials/effective-set.md new file mode 100644 index 000000000..e70a4bde9 --- /dev/null +++ b/docs/tutorials/effective-set.md @@ -0,0 +1,467 @@ +# Tutorial: Understanding the Effective Set + +- [Tutorial: Understanding the Effective Set](#tutorial-understanding-the-effective-set) + - [What You Will Learn](#what-you-will-learn) + - [Prerequisites](#prerequisites) + - [Scenario](#scenario) + - [Step 1: Understand What the Effective Set Is](#step-1-understand-what-the-effective-set-is) + - [Step 2: Generate the Effective Set](#step-2-generate-the-effective-set) + - [Step 3: Read the Deployment Context](#step-3-read-the-deployment-context) + - [Step 4: Trace How Parameters Flow Into the Effective Set](#step-4-trace-how-parameters-flow-into-the-effective-set) + - [4.1 Resource sizing in per-service-parameters](#41-resource-sizing-in-per-service-parameters) + - [Step 5: Read the Topology Context](#step-5-read-the-topology-context) + - [Step 6: Read the Pipeline Context](#step-6-read-the-pipeline-context) + - [Step 7: Understand Parameter Priority](#step-7-understand-parameter-priority) + - [7.1 Verify priority with traceability comments](#71-verify-priority-with-traceability-comments) + - [Step 8: Understand What Triggers a Regeneration](#step-8-understand-what-triggers-a-regeneration) + - [Summary](#summary) + +## What You Will Learn + +By the end of this tutorial you will know how to: + +- Explain what the Effective Set is, why it exists, and what its contexts are +- Generate the Effective Set with traceability enabled +- Read the deployment context: understand the values files, their priority order, and traceability comments +- Trace how parameters from Tenant, Cloud, Namespace, Application, SBOM, and Resource Profile sources end up in the Effective Set +- Read the topology and pipeline contexts and understand what each one is for +- Use the parameter priority order and traceability comments to debug a wrong value +- Identify what changes require a regeneration + +## Prerequisites + +- A working Instance Repository with at least one environment that has been through `env_build` +- A Solution Descriptor present at `environments///Inventory/solution-descriptor/sd.yaml` +- Basic familiarity with EnvGene Environment Instance structure (Tenant, Cloud, Namespace objects) + +## Scenario + +You manage a solution called BSS deployed to `prod-cluster/prod-01`. The solution has two namespaces: + +- `bss` - hosts the `Cloud-BSS` application with two services: `bss-processor` and `bss-api` +- `oss` - hosts the `cloud-oss` application with one service: `oss-api` + +You want to understand how the Effective Set is calculated for `prod-cluster/prod-01`, where each value comes from, and how to read the output files to debug a deployment issue. + +## Step 1: Understand What the Effective Set Is + +The Effective Set is the **final, merged parameter file tree** that EnvGene calculates for one environment. It is not something you write manually - EnvGene generates it by merging many input sources in a defined priority order. + +To understand why it exists, consider the problem it solves. Configuration for one environment is spread across many sources: + +- **Template Repository** - Namespace templates, Cloud templates, Template ParameterSets, Resource Profile Overrides. + +- **Instance Repository** - Cloud and Namespace objects with environment-specific parameters, Environment Specific ParameterSets, Resource Profile Overrides, Credentials, Cloud Passport. The Instance stores configuration for the environment itself but does not specify which applications are deployed to which namespaces or at which versions - that is the job of the Solution Descriptor. + +- **Solution Descriptor (SD)** - maps applications to namespaces and defines which version of each application to deploy. Each entry pairs an application version with a `deployPostfix` that identifies the target namespace. For `prod-01` this looks like: + + ```yaml + applications: + - version: "Cloud-BSS:1.2.3" + deployPostfix: "bss" + - version: "cloud-oss:2.0.1" + deployPostfix: "oss" + ``` + + This tells EnvGene that `Cloud-BSS v1.2.3` deploys to the `bss` namespace and `cloud-oss v2.0.1` to the `oss` namespace. Without an SD, EnvGene does not know what applications exist and cannot generate the `deployment`, `cleanup`, or `runtime` contexts. + +- **Application SBOMs** - for each application version listed in the SD, EnvGene reads the corresponding SBOM from `sboms//` (filename: `-.sbom.json`). The SBOM is an EnvGene-internal artifact generated automatically per application version. Examples of what it includes: + - The list of microservices the application consists of (determines the structure of `per-service-parameters/`) + - Docker image coordinates for each microservice (written to `deploy-descriptor.yaml`) + - Resource Profile Baselines - named sets of CPU/memory/replica values that serve as the starting point before any overrides are applied. + +No single downstream tool can read all of these sources, understand their merge rules, and resolve Jinja expressions by itself. Instead, EnvGene acts as a pre-processor: it reads all sources, applies all merge rules, and writes a single output - the Effective Set - that contains only final, resolved values with no templates, no references, no indirection. + +The Effective Set for `prod-cluster/prod-01` lives at: + +```text +environments/prod-cluster/prod-01/effective-set/ +``` + +It is divided into five **contexts**. Each context has a format specific to its consumer - a tool reads only its own context and must not access other contexts. This means a change to any input that has **not been followed by a regeneration** is invisible to all consumers. + +> [!NOTE] +> The SD is optional inputs. Without an SD, the `deployment`, `cleanup`, and `runtime` contexts are not generated - only `topology` and `pipeline` are produced. + +| Context | Path | Source in Instance objects | Consumer | +|--------------|-----------------------------|--------------------------------------|-------------------------------| +| `deployment` | `effective-set/deployment/` | `deployParameters`, Application SBOM | ArgoCD, Helm | +| `pipeline` | `effective-set/pipeline/` | `e2eParameters` | Orchestration pipelines | +| `runtime` | `effective-set/runtime/` | `technicalConfigurationParameters` | Runtime configuration tooling | +| `cleanup` | `effective-set/cleanup/` | `deployParameters` | Uninstall tooling | +| `topology` | `effective-set/topology/` | Aggregated environment data | All consumers | + +**deployment** - Helm values applied at deploy time: infrastructure service URLs (Consul, DBaaS, MaaS), platform flags, monitoring settings, application-specific configuration, image tags, resource profile baselines (CPU, memory, replicas). + +**pipeline** - Parameters that configure how the orchestration pipeline operates for this environment: deployment tool URLs, per-namespace deployment policies, notification channels, test system settings, pipeline job references. + +**runtime** - Parameters that configure application behavior while it is already running. Applied via Consul without redeployment, allowing live configuration changes without a new Helm release. + +**cleanup** - Parameters needed during uninstall and teardown operations. + +**topology** - A structural description of the environment: which namespaces exist, how they relate (composite structure, BG domain), and cluster connection details. Unlike the other contexts, `topology` is not tied to a single consumer - any tool that needs to understand the environment layout may read it. + +Each context that contains sensitive parameters includes a `credentials.yaml` alongside the main values file. `credentials.yaml` holds parameters whose values are defined using credential macros in the Environment Instance - those values are resolved and SOPS-encrypted in the output. The unencrypted values are never written to disk. + +In this tutorial you will focus on the `deployment`, `topology`, and `pipeline` contexts, which are the most commonly used. + +## Step 2: Generate the Effective Set + +Trigger the Instance pipeline with the following variables: + +```text +ENV_NAMES: prod-cluster/prod-01 +GENERATE_EFFECTIVE_SET: true +EFFECTIVE_SET_CONFIG: {"enable_traceability":true} +``` + +`EFFECTIVE_SET_CONFIG` is a JSON object. In the pipeline UI you enter it as-is: `{"enable_traceability":true}`. In a YAML file (e.g. `.gitlab-ci.yml`) the value must be quoted and the inner quotes escaped: `"{\"enable_traceability\":true}"`. All fields are optional: + +| Field | Default | Description | +|---------------------------------|----------|-----------------------------------------------------------------| +| `version` | `v2.0` | Effective Set version. `v1.0` is legacy | +| `enable_traceability` | `false` | Add source comments showing each parameter's origin object type | +| `app_chart_validation` | `true` | Fail if any application is not an app chart application | +| `effective_set_expiry` | `1 hour` | CI artifact retention time | +| `contexts.pipeline.consumers[]` | none | Consumer-specific pipeline context schemas | + +`contexts.pipeline.consumers` adds consumer-specific pipeline sub context files alongside the generic `pipeline/parameters.yaml`. For example, adding `{"name":"dcl","version":"v1.0"}` produces `pipeline/dcl-parameters.yaml` and `pipeline/dcl-credentials.yaml`. See [Instance Pipeline Parameters](/docs/instance-pipeline-parameters.md#effective_set_config) for the full schema. + +With `GENERATE_EFFECTIVE_SET: true` set, the pipeline runs: + +```text +generate_effective_set → git_commit +``` + +`generate_effective_set` reads the Environment Instance files that are already in the repository (produced by a previous `env_build` run) and writes the Effective Set on top of them. It does not rebuild the Environment Instance itself. + +After the pipeline completes, pull the latest changes from the repository: + +```bash +git pull origin main +``` + +The generated files are now committed at `environments/prod-cluster/prod-01/effective-set/`. The directory structure for our scenario looks like this: + +```text +effective-set/ +├── deployment/ +│ ├── mapping.yml +│ ├── bss/ +│ │ └── Cloud-BSS/ +│ │ └── values/ +│ │ ├── collision-credentials.yaml +│ │ ├── collision-deployment-parameters.yaml +│ │ ├── credentials.yaml +│ │ ├── custom-params.yaml +│ │ ├── deploy-descriptor.yaml +│ │ ├── deployment-parameters.yaml +│ │ └── per-service-parameters/ +│ │ └── cloud-bss-7b/ +│ │ └── deployment-parameters.yaml +│ └── oss/ +│ └── cloud-oss/ +│ └── values/ +│ ├── collision-credentials.yaml +│ ├── collision-deployment-parameters.yaml +│ ├── credentials.yaml +│ ├── custom-params.yaml +│ ├── deploy-descriptor.yaml +│ ├── deployment-parameters.yaml +│ └── per-service-parameters/ +│ └── cloud-oss/ +│ └── deployment-parameters.yaml +├── topology/ +│ ├── credentials.yaml +│ └── parameters.yaml +├── pipeline/ +│ ├── credentials.yaml +│ └── parameters.yaml +├── runtime/ +│ ├── mapping.yml +│ ├── bss/ +│ │ └── Cloud-BSS/ +│ │ ├── credentials.yaml +│ │ └── parameters.yaml +│ └── oss/ +│ └── cloud-oss/ +│ ├── credentials.yaml +│ └── parameters.yaml +└── cleanup/ + ├── mapping.yml + ├── bss/ + │ ├── credentials.yaml + │ └── parameters.yaml + └── oss/ + ├── credentials.yaml + └── parameters.yaml +``` + +`mapping.yml` appears in the `deployment/`, `runtime/`, and `cleanup/` contexts. It maps each namespace name to its folder path within the context, so consumers can locate the right directory without inferring naming conventions. For example, a consumer looking up parameters for the `prod-01-bss` namespace reads `mapping.yml` to find the folder `bss/`. + +The `per-service-parameters/` subfolder name is application name normalized to comply with Kubernetes naming rules. `Cloud-BSS` becomes `cloud-bss-7b`; `cloud-oss` is already lowercase and stays unchanged. [Section 4.1](#41-resource-sizing-in-per-service-parameters) explains this in detail. + +> [!NOTE] +> This tutorial simplifies two things compared to real usage: +> +> - The SD is already committed to the repository, so no SD parameters are needed. +> - Only `generate_effective_set` is triggered; `env_build` does not run. +> +> In practice, Effective Set generation is a pre-step before deployment. +> A deployment typically means a new application version or a new application being added - the SD carries exactly this information: which applications to deploy and at which versions. +> At the same time, `env_build` ensures the Environment Instance reflects the latest template version before the Effective Set is calculated. +> These steps therefore go together in a single run: +> +> ```text +> ENV_NAMES: prod-cluster/prod-01 +> ENV_BUILDER: true +> ENV_TEMPLATE_VERSION: env-template:2.5.0 +> GENERATE_EFFECTIVE_SET: true +> EFFECTIVE_SET_CONFIG: {"enable_traceability":true} +> SD_SOURCE_TYPE: artifact +> SD_VERSION: sd:1.2.3 +> ``` + +## Step 3: Read the Deployment Context + +The deployment context is organized by namespace and application - each folder name corresponds to the `deployPostfix` value from the SD: + +```text +effective-set/deployment/ +└── / ← namespace folder, e.g. bss + └── / ← application name, e.g. Cloud-BSS + └── values/ ← Helm values files applied in priority order +``` + +Navigate to the `Cloud-BSS` application folder: + +```text +environments/prod-cluster/prod-01/effective-set/deployment/bss/Cloud-BSS/values/ +``` + +For each application the deployer receives a set of values files applied in this priority order (last file wins): + +```text +deploy-descriptor.yaml ← SBOM-derived artifact metadata (image names, tags, chart coordinates) +credentials.yaml ← sensitive user defined params; SOPS-encrypted if configured +deployment-parameters.yaml ← user defined params +collision-deployment-parameters.yaml ← params whose key matches a service name (moved to avoid Helm structure conflicts) +collision-credentials.yaml ← sensitive collision params; SOPS-encrypted if configured +per-service-parameters//deployment-parameters.yaml ← per-microservice resource sizing (all services as keys) +custom-params.yaml ← CUSTOM_PARAMS pipeline variable; highest priority +``` + +`deploy-descriptor.yaml` is generated entirely from the Application SBOM. It contains artifact metadata - Docker image names and tags, Helm chart coordinates, Git revision. It is read-only; users cannot override its values. + +With `enable_traceability: true`, every parameter in `deployment-parameters.yaml` carries an inline comment identifying its source object type. [Step 4](#step-4-trace-how-parameters-flow-into-the-effective-set) traces these in detail. + +## Step 4: Trace How Parameters Flow Into the Effective Set + +With traceability enabled, every parameter in `deployment-parameters.yaml` carries a comment that identifies its source object type. This is the primary tool for debugging a wrong value: find the parameter, read the comment, go to that object. + +Open `environments/prod-cluster/prod-01/effective-set/deployment/bss/Cloud-BSS/values/deployment-parameters.yaml`: + +```yaml +ARGOCD_URL: "https://argocd.prod-cluster.example.com" #cloud +PAAS_PLATFORM: "KUBERNETES" #cloud +BSS_NAMESPACE: "prod-01-bss" #namespace +CUSTOM_HOST: "bss.prod-cluster.example.com" #namespace +SERVICE_TYPE: "ClusterIP" #application +HEALTH_CHECK_PATH: "/api/health" #application +MANAGED_BY: "argocd" #envgene default +``` + +Each comment tells you where to look: + +| Comment | Source object | Where to look in the Instance | +|-----------------------|-------------------------|-----------------------------------------------------| +| `#tenant` | Tenant object | `tenant.yml` or its ParameterSets | +| `#cloud` | Cloud object | `cloud.yml` or its ParameterSets | +| `#namespace` | Namespace object | `Namespaces/bss/namespace.yml` or its ParameterSets | +| `#application` | Application object | `Namespaces/bss/Applications/Cloud-BSS.yml` | +| `#sbom` | Application SBOM | SBOM in `sboms/` - not directly editable | +| `#envgene calculated` | Computed by EnvGene | Not editable - derived value | +| `#envgene default` | EnvGene built-in | Internal default - not configurable | +| `#custom params` | `CUSTOM_PARAMS` var | Pipeline variable - highest priority | + +For the complete list of comment types see the [Calculator CLI Reference](../features/calculator-cli.md#version-20-traceability-comments). + +### 4.1 Resource sizing in per-service-parameters + +The `per-service-parameters/cloud-bss-7b/deployment-parameters.yaml` file uses different comment types because its values come from the SBOM and Resource Profile Overrides: + +```yaml +bss-api: + CPU_REQUEST: 250m #rp-baseline: prod + CPU_LIMIT: 500m #rp-baseline: prod + REPLICAS: 2 #rp-baseline: prod +bss-processor: + CPU_REQUEST: 500m #rp-baseline: prod + CPU_LIMIT: 2000m #rp-baseline: prod + REPLICAS: 5 #rp-override: prod-bss-override +``` + +`#rp-baseline: prod` - value comes from the Resource Profile Baseline named `prod` in the application SBOM. To change it, add a Resource Profile Override in the Instance. + +`#rp-override: prod-bss-override` - the SBOM baseline was overridden by `Profiles/prod-bss-override.yml` in the Instance. + +> [!NOTE] +> If the application is not an app chart (its Helm chart is a flat chart, not an umbrella chart with nested sub-charts), the structure differs: each microservice gets its own subfolder with a flat (non-nested) `deployment-parameters.yaml`. See [Calculator CLI Reference](/docs/features/calculator-cli.md) for details. + +## Step 5: Read the Topology Context + +Open: + +```text +effective-set/topology/parameters.yaml +``` + +The topology context describes the environment's structural layout. It is not per-application - it covers the entire environment: + +```yaml +cluster: + api_url: api.prod-cluster.example.com + api_port: "6443" + protocol: HTTPS + public_url: prod-cluster.example.com + +environments: + prod-cluster/prod-01: + namespaces: + prod-01-bss: + deployPostfix: bss + prod-01-oss: + deployPostfix: oss + +composite_structure: {} + +bg_domain: {} +``` + +The topology context is consumed by orchestration tools that need to understand which namespaces belong to an environment, how they relate to each other, and cluster connection details. `composite_structure` and `bg_domain` are empty when not configured for this environment. + +The `topology/credentials.yaml` file contains per-namespace Kubernetes service account tokens (SOPS-encrypted): + +```yaml +k8s_tokens: + prod-01-bss: ENC[AES256_GCM,data:...] + prod-01-oss: ENC[AES256_GCM,data:...] +``` + +## Step 6: Read the Pipeline Context + +Open: + +```text +effective-set/pipeline/parameters.yaml +``` + +The pipeline context contains parameters consumed by orchestration pipelines - values that control how downstream pipeline jobs run rather than how the application itself is configured: + +```yaml +DCL_CONFIG_ARGOCD_URL: "https://argocd-server.prod-cluster.example.com" +DCL_CONFIG_ARGOCD_PROJECT: "infra" +DCL_CONFIG_DOCKER_REGISTRY: "registry.example.com:17014" +DCL_CONFIG_REGISTRY_URL: "https://registry.example.com" +DCL_CONFIG_SKIP_CREDENTIALS_ENCRYPTION: false +DCL_CONFIG_SOPS_DCL2ARGO_AGE_PUBLIC_KEY: "age1abc123..." +DCL_GIT_BRANCH: master +DCL_GIT_URL: "https://git.example.com/pipelines/gitlab-dcl" +``` + +These parameters come from the `e2eParameters` sections of the Cloud object in the Environment Instance. Sensitive values (ArgoCD credentials, SSO secrets) are split into `pipeline/credentials.yaml` which is SOPS-encrypted. + +> [!NOTE] +> The attribute is called `e2eParameters` for historical reasons. They are general-purpose pipeline orchestration parameters. + +If consumer-specific contexts were configured in `EFFECTIVE_SET_CONFIG`, their files appear alongside the generic ones: + +```text +pipeline/ +├── parameters.yaml +├── credentials.yaml +├── e2e-runner-parameters.yaml ← consumer-specific +└── e2e-runner-credentials.yaml ← consumer-specific +``` + +The consumer-specific files contain a validated subset of the pipeline parameters, filtered through the consumer's JSON schema. + +## Step 7: Understand Parameter Priority + +When the same parameter key is defined at multiple levels, EnvGene applies a strict priority order to decide which value wins. From highest to lowest: + +1. **Custom Params** (`CUSTOM_PARAMS` pipeline variable) - highest, always wins +2. **Resource Profile Override** - instance-level (in `Profiles/`) takes precedence over template-level +3. **Resource Profile Baseline** (from SBOM) +4. **Application object** (`deployParameters` in `Applications/.yml`) +5. **Namespace object** (`deployParameters` in `namespace.yml`) +6. **Cloud object** (`deployParameters` in `cloud.yml`) +7. **Tenant object** (`deployParameters` in `tenant.yml`) + +A practical example: if `Tenant` defines `LOG_LEVEL: INFO` and `Namespace` defines `LOG_LEVEL: DEBUG`, the value in `deployment-parameters.yaml` will be `DEBUG` because Namespace has higher priority than Tenant. + +### 7.1 Verify priority with traceability comments + +Suppose `deployment-parameters.yaml` for `Cloud-BSS` shows: + +```yaml +LOG_LEVEL: "DEBUG" #namespace +``` + +The comment `#namespace` tells you the value is defined at the namespace level - either directly in `namespace.yml` or in a ParameterSet referenced by it. To change it, check `namespace.yml`: + +```text +environments/prod-cluster/prod-01/Namespaces/bss/namespace.yml +``` + +or the ParameterSet files under `Inventory/parameters/` that are assigned to the `bss` namespace. After editing, regenerate the Effective Set. + +The same logic applies to other comments: + +| Comment | Where to look | +|--------------------------|-----------------------------------------------------------| +| `#tenant` | `tenant.yml` or ParameterSets assigned to Tenant | +| `#cloud` | `cloud.yml` or ParameterSets assigned to Cloud | +| `#namespace` | `namespace.yml` or ParameterSets assigned to Namespace | +| `#application` | Application object in `Applications/` | +| `#sbom` | Application SBOM in `sboms/` - not directly editable | +| `#rp-baseline: ` | SBOM baseline - override with a Resource Profile Override | +| `#rp-override: ` | `Profiles/` in the Instance or Template Repository | +| `#envgene calculated` | Derived by EnvGene - not directly editable | +| `#envgene default` | Internal default - not configurable | +| `#custom params` | `CUSTOM_PARAMS` pipeline variable - highest priority | + +## Step 8: Understand What Triggers a Regeneration + +The Effective Set becomes stale whenever any of its inputs change. You must regenerate it in these situations: + +| Change | Required action | +|------------------------------------------------------|--------------------------------------------| +| Environment Specific ParameterSet modified | Regenerate Effective Set | +| Template ParameterSet changed (new template version) | Regenerate both Instance and Effective Set | +| Resource Profile Override modified | Regenerate Effective Set | +| Solution Descriptor applications changed | Update SD, regenerate Effective Set | +| Cloud Passport data updated | Regenerate both Instance and Effective Set | +| Credentials rotated | Regenerate Effective Set | + +## Summary + +In this tutorial you traced the full Effective Set lifecycle for the BSS solution: + +- **What it is** - the final merged parameter tree for one environment, written to `effective-set/` and read by ArgoCD and other consumers. +- **Five contexts** - `deployment` (application parameters and artifacts), `pipeline` (CI/CD parameters), `topology` (cluster layout and structure), `runtime` (runtime management parameters), `cleanup` (teardown parameters). +- **Parameter flow** - Tenant → Cloud → Namespace → Application → SBOM → Resource Profile Override, with each level overriding the previous one. +- **deployment-parameters.yaml** - the merged application parameters; traceability comments show which object type (Cloud, Namespace, Application, etc.) contributed each value. +- **deploy-descriptor.yaml** - SBOM-derived artifact metadata consumed by the deployer to know which Docker images and charts to install. +- **per-service-parameters** - resource sizing per microservice; `#rp-baseline` and `#rp-override` comments identify the source. +- **topology/parameters.yaml** - cluster structure: environment list, namespace mapping, composite structure, BG domain. +- **Priority order** - Custom Params win over everything; Instance ParameterSets override Template ParameterSets; Namespace overrides Cloud overrides Tenant. +- **Regeneration triggers** - any change to inputs (ParameterSets, Resource Profiles, SBOMs, SD, credentials) requires a new Effective Set generation. + +**Related documentation:** + +- [Calculator CLI](/docs/features/calculator-cli.md) +- [How to Generate an Effective Set](/docs/how-to/generate-effective-set.md) +- [How to Override Template Parameters](/docs/how-to/environment-specific-parameters.md) +- [Tutorial: Managing Resource Profiles](/docs/tutorials/resource-profiles.md) +- [Instance Pipeline Parameters](/docs/instance-pipeline-parameters.md) diff --git a/docs/tutorials/envgene-null-value.md b/docs/tutorials/envgene-null-value.md new file mode 100644 index 000000000..c384ffb96 --- /dev/null +++ b/docs/tutorials/envgene-null-value.md @@ -0,0 +1,200 @@ +# Working with envgeneNullValue + +- [Working with envgeneNullValue](#working-with-envgenenullvalue) + - [What You Will Learn](#what-you-will-learn) + - [Prerequisites](#prerequisites) + - [Overview](#overview) + - [Scenario: Credentials Placeholder](#scenario-credentials-placeholder) + - [Problem](#problem) + - [Credential Type 1: usernamePassword](#credential-type-1-usernamepassword) + - [Generated `credentials.yml` (username/password)](#generated-credentialsyml-usernamepassword) + - [Behavior When Values Are Missing](#behavior-when-values-are-missing) + - [Credential Type 2: secret](#credential-type-2-secret) + - [Generated `credentials.yml` (secret)](#generated-credentialsyml-secret) + - [Behavior When Value Is Missing](#behavior-when-value-is-missing) + - [How to Resolve Credentials](#how-to-resolve-credentials) + - [Option 1: Cloud Passport](#option-1-cloud-passport) + - [Option 2: Shared Credentials](#option-2-shared-credentials) + - [usernamePassword example](#usernamepassword-example) + - [secret example](#secret-example) + - [Verification Step (Required)](#verification-step-required) + - [Before / After Example](#before--after-example) + - [Summary](#summary) + +## What You Will Learn + +By the end of this tutorial you will understand: + +- What `envgeneNullValue` is +- Why EnvGene uses it +- A **practical scenario: credentials placeholder** covering: + + - `usernamePassword` credentials + - `secret` credentials + +## Prerequisites + +- A working EnvGene setup (Template Repository + Instance Repository) +- Basic understanding of environment generation + +## Overview + +`envgeneNullValue` is a special placeholder used by EnvGene to represent **missing or unresolved values**. + +It is intentionally used to: + +- Mark values that must be provided later +- Prevent incomplete or insecure deployments + +Common use case: + +If a required value remains `envgeneNullValue` where a real value is mandatory, validation fails and deployment is blocked. + +## Scenario: Credentials Placeholder + +### Problem + +When EnvGene fills the [Environment Credentials File](/docs/envgene-objects.md#environment-credentials-file) (`Credentials/credentials.yml`), +it not have access to actual secret values. + +Instead, credential fields can be set to `envgeneNullValue` until you resolve them. + +## Credential Type 1: usernamePassword + +### Generated `credentials.yml` (username/password) + +```yaml +dbaas-cluster-dba-cred: + type: "usernamePassword" + data: + username: "envgeneNullValue" # FillMe + password: "envgeneNullValue" # FillMe +``` + +### Behavior When Values Are Missing + +If credentials are not resolved: + +- Validation fails during environment generation +- Deployment is blocked + +Example error: + +```text +envgenehelper.errors.ValidationError: Error while validating credentials: + credId: dbaas-cluster-dba-cred - username or password is not set +``` + +## Credential Type 2: secret + +### Generated `credentials.yml` (secret) + +```yaml +consul-admin-cred: + type: "secret" + data: + secret: "envgeneNullValue" # FillMe +``` + +### Behavior When Value Is Missing + +If the secret is not resolved: + +- Validation fails during environment generation +- Deployment is blocked + +Example error: + +```text +Error while validating credentials: + credId: consul-admin-cred - secret is not set +``` + +## How to Resolve Credentials + +`envgeneNullValue` must be replaced before deployment using one of the supported methods. + +### Option 1: Cloud Passport + +Provide credential values via Cloud Passport configuration. + +### Option 2: Shared Credentials + +See [Shared Credentials File](/docs/envgene-objects.md#shared-credentials-file) in `envgene-objects.md` for locations and merge behavior. + +#### usernamePassword example + +```yaml +dbaas-cluster-dba-cred: + type: "usernamePassword" + data: + username: "real_user" + password: "secure_password" +``` + +#### secret example + +```yaml +consul-admin-cred: + type: "secret" + data: + secret: "secret-123" +``` + +## Verification Step (Required) + +Before deployment, ensure no placeholders remain. + +PowerShell example: + +```powershell +Get-ChildItem -Recurse -Include *.yml,*.yaml | + Select-String -Pattern 'envgeneNullValue' +``` + +If any matches are found: + +- Do **not** proceed with deployment + +## Before / After Example + +**Before resolution** (same structure as the generated `credentials.yml` examples above): + +```yaml +dbaas-cluster-dba-cred: + type: "usernamePassword" + data: + username: "envgeneNullValue" # FillMe + password: "envgeneNullValue" # FillMe + +consul-admin-cred: + type: "secret" + data: + secret: "envgeneNullValue" # FillMe +``` + +**After resolution:** + +```yaml +dbaas-cluster-dba-cred: + type: "usernamePassword" + data: + username: "prod_user" + password: "secure_password" + +consul-admin-cred: + type: "secret" + data: + secret: "secret-123" +``` + +## Summary + +- `envgeneNullValue` is an **intentional placeholder** +- Used in **credentials generation** for: + + - `usernamePassword` + - `secret` types + +- Prevents incomplete or insecure deployments when validation requires real values +- Must be replaced with real values before deployment wherever your pipeline enforces it diff --git a/docs/tutorials/resource-profiles.md b/docs/tutorials/resource-profiles.md new file mode 100644 index 000000000..6804d6721 --- /dev/null +++ b/docs/tutorials/resource-profiles.md @@ -0,0 +1,463 @@ +# Tutorial: Managing Resource Profiles + +- [Tutorial: Managing Resource Profiles](#tutorial-managing-resource-profiles) + - [What You Will Learn](#what-you-will-learn) + - [Prerequisites](#prerequisites) + - [Scenario](#scenario) + - [Step 1: Understand the Resource Profile Hierarchy](#step-1-understand-the-resource-profile-hierarchy) + - [Step 2: Create a Template Resource Profile Override](#step-2-create-a-template-resource-profile-override) + - [2.1 Create the override file](#21-create-the-override-file) + - [2.2 Reference the profile in a Namespace template](#22-reference-the-profile-in-a-namespace-template) + - [Step 3: Use `template_override` to Differentiate Template Profiles](#step-3-use-template_override-to-differentiate-template-profiles) + - [Step 4: Use `overrides-parent` in a Composed Template](#step-4-use-overrides-parent-in-a-composed-template) + - [Step 5: Add an Environment-Specific Override in the Instance Repository](#step-5-add-an-environment-specific-override-in-the-instance-repository) + - [5.1 Choose the right scope](#51-choose-the-right-scope) + - [5.2 Create the env-specific override file](#52-create-the-env-specific-override-file) + - [5.3 Reference the override in `env_definition.yml`](#53-reference-the-override-in-env_definitionyml) + - [Step 6: Override the Override for a Single Environment](#step-6-override-the-override-for-a-single-environment) + - [Step 7: Verify the Result](#step-7-verify-the-result) + - [7.1 Profile Override file in `Profiles/`](#71-profile-override-file-in-profiles) + - [7.2 Effective Set](#72-effective-set) + - [Summary](#summary) + +## What You Will Learn + +By the end of this tutorial you will know how to: + +- Read and understand a Resource Profile Baseline distributed with an application artifact +- Create a Template Resource Profile Override in the Template Repository +- Apply different profiles to different environment types without duplicating templates +- Add an environment-specific override in the Instance Repository +- Override the override for a single environment + +## Prerequisites + +- A working Template Repository with Cloud and at least one Namespace template +- A working Instance Repository with at least one environment +- Basic familiarity with EnvGene template and instance repository structure + +## Scenario + +You manage a solution called BSS that consists of one application (`Cloud-BSS`) with a service called `bss-processor`. The solution is deployed to two types of environments: `dev` and `prod`. You want to: + +1. Apply modest, developer-friendly resource limits to all dev environments. +2. Apply production-grade resource limits to all prod environments. +3. Temporarily double the CPU limit for a single environment called `prod-01` during a load test. + +## Step 1: Understand the Resource Profile Hierarchy + +EnvGene manages performance parameters (CPU, memory, replicas, etc.) through a three-level hierarchy: + +| Level | Name | Where it lives | Who controls it | +|-------|----------------------------------------------------|------------------------------------------------------------|-----------------------| +| 1 | Resource Profile **Baseline** | Application SBOM artifact | Application developer | +| 2 | **Template** Resource Profile Override | Template Repository `/templates/resource_profiles/` | Template configurator | +| 3 | **Environment-Specific** Resource Profile Override | Instance Repository `/environments/.../resource_profiles/` | Instance configurator | + +Each level overrides the previous one. The Baseline is the starting point; it ships inside the application (SBOM) and is selected by the name defined in the Cloud or Namespace template (e.g. `dev`, `prod`). You do not edit it. + +A typical Baseline embedded in the application looks like this (shown for orientation only - you cannot change it): + +```yaml +# Baseline name: "dev" +CPU_REQUEST: 100m +CPU_LIMIT: 500m +MEMORY_REQUEST: 256Mi +MEMORY_LIMIT: 512Mi +REPLICAS: 1 +``` + +> [!NOTE] +> EnvGene also supports dot-notation keys (e.g. `resources.limits.cpu`) in Baselines and Override files +> and resolves them into nested YAML structures. + +## Step 2: Create a Template Resource Profile Override + +A Template Resource Profile Override lets you tune the Baseline values for all environments that use the same template - without touching the application artifact itself. + +### 2.1 Create the override file + +Create two files in the Template Repository, one for `dev` and one for `prod`: + +**`/templates/resource_profiles/dev-bss-override.yml`** + +```yaml +name: "dev-bss-override" +baseline: "dev" +description: "Dev resource profile for BSS processor" +applications: + - name: "Cloud-BSS" + services: + - name: "bss-processor" + parameters: + - name: "CPU_REQUEST" + value: "100m" + - name: "CPU_LIMIT" + value: "500m" + - name: "MEMORY_REQUEST" + value: "256Mi" + - name: "MEMORY_LIMIT" + value: "512Mi" + - name: "REPLICAS" + value: 1 +``` + +**`/templates/resource_profiles/prod-bss-override.yml`** + +```yaml +name: "prod-bss-override" +baseline: "prod" +description: "Production resource profile for BSS processor" +applications: + - name: "Cloud-BSS" + services: + - name: "bss-processor" + parameters: + - name: "CPU_REQUEST" + value: "2000m" + - name: "CPU_LIMIT" + value: "4000m" + - name: "MEMORY_REQUEST" + value: "2Gi" + - name: "MEMORY_LIMIT" + value: "4Gi" + - name: "REPLICAS" + value: 5 +``` + +> The `name` field **must** exactly match the filename without the extension. +> The `baseline` field is informational only and is not processed by EnvGene. + +### 2.2 Reference the profile in a Namespace template + +Open your BSS Namespace template and add the `profile` section: + +**`/templates/env_templates/dev/Namespaces/bss.yml.j2`** + +```yaml +--- +name: "{{ current_env.environmentName }}-bss" +# ... other required fields ... +profile: + name: dev-bss-override + baseline: dev +# ... other required fields ... +``` + +**`/templates/env_templates/prod/Namespaces/bss.yml.j2`** + +```yaml +--- +name: "{{ current_env.environmentName }}-bss" +# ... other required fields ... +profile: + name: prod-bss-override + baseline: prod +# ... other required fields ... +``` + +EnvGene reads `profile.name` from the rendered template and looks up the file `.yml` (or `.yaml`) inside `/templates/resource_profiles/`. + +## Step 3: Use `template_override` to Differentiate Template Profiles + +The two-file approach from Step 2 works, but maintaining separate template files that differ only in the `profile` section does not scale. An alternative is to keep a **single**, profile-neutral base template and assign the profile at the descriptor level using `template_override`. + +Replace the two namespace templates from Step 2 with one shared file: + +**`/templates/env_templates/base/Namespaces/bss.yml.j2`** + +```yaml +--- +name: "{{ current_env.environmentName }}-bss" +# ... other required fields ... +# no profile section +``` + +Then create two Template Descriptors that reference it with different profiles: + +**`/templates/env_templates/dev.yml`** + +```yaml +tenant: "{{ templates_dir }}/env_templates/base/tenant.yml.j2" +cloud: "{{ templates_dir }}/env_templates/base/cloud.yml.j2" +namespaces: + - template_path: "{{ templates_dir }}/env_templates/base/Namespaces/bss.yml.j2" + template_override: + profile: + name: "dev-bss-override" + baseline: "dev" +``` + +**`/templates/env_templates/prod.yml`** + +```yaml +tenant: "{{ templates_dir }}/env_templates/base/tenant.yml.j2" +cloud: "{{ templates_dir }}/env_templates/base/cloud.yml.j2" +namespaces: + - template_path: "{{ templates_dir }}/env_templates/base/Namespaces/bss.yml.j2" + template_override: + profile: + name: "prod-bss-override" + baseline: "prod" +``` + +When EnvGene generates an Environment Instance it renders `bss.yml.j2`, then merges the `template_override` block into the result. The final Namespace object gets `profile.name: dev-bss-override` (or `prod-bss-override`), overriding whatever the template originally defined. + +`template_override` supports Jinja expressions, so you can also compute profile names dynamically from environment variables. + +## Step 4: Use `overrides-parent` in a Composed Template + +A different situation arises when you consume a component from an **external artifact** - a parent template referenced via `parent-templates` that you cannot edit directly. `overrides-parent` is the right mechanism for this case: it lets a child Template Descriptor swap or augment the parent's profile without touching the parent artifact. + +Suppose a parent template (`bss-product-template:2.0.0`) ships with a namespace whose `profile.name` is `default-bss-override`. A child (project-level) template wants to apply `project-bss-override` instead. + +In the **child** Template Descriptor: + +```yaml +parent-templates: + default-bss: bss-product-template:2.0.0 + +namespaces: + - name: "{env}-bss" + parent: default-bss + overrides-parent: + profile: + override-profile-name: "project-bss-override" + parent-profile-name: "default-bss-override" + baseline-profile-name: "dev" + merge-with-parent: true +``` + +Key fields explained: + +| Field | Required | Description | +|-------------------------|----------|-------------------------------------------------------| +| `override-profile-name` | Optional | Profile file in the **child** template repository | +| `parent-profile-name` | Optional | Profile from the **parent** template to override | +| `baseline-profile-name` | Optional | Baseline name to set | +| `merge-with-parent` | Optional | `true`: merge into parent; `false`: replace (default) | + +When `merge-with-parent: true` the child's `project-bss-override` values are **merged into** the parent's `default-bss-override`. Only the parameters listed in `project-bss-override` are overwritten; all other parent parameters are preserved. + +When `merge-with-parent: false` the parent's profile is completely replaced by `project-bss-override`. + +The `overrides-parent` mechanism works for Cloud templates too - place the same `profile` sub-block under `cloud.overrides-parent`. + +## Step 5: Add an Environment-Specific Override in the Instance Repository + +You can further adjust resource parameters for a specific environment in the Instance Repository without modifying the Template Repository at all. + +### 5.1 Choose the right scope + +Place the override file in the location that matches the desired scope: + +| Location | Scope | Use When | +|-----------------------------------------------------------------|-----------------|------------------------------| +| `/environments///Inventory/resource_profiles/` | One environment | Env-specific tuning | +| `/environments//resource_profiles/` | Cluster-wide | All envs in cluster | +| `/environments/resource_profiles/` | Global | All envs in the repository | + +When an environment references a file by name (via `envSpecificResourceProfiles`), EnvGene searches these three locations **from most specific to most general** and uses the first file it finds with that name. + +**How to decide:** + +Ask yourself: *which environments need this override?* + +- **One specific environment** - use the environment-specific path. Changes stay isolated and do not affect any neighbor +- **All environments in a cluster** - use the cluster-wide path. One file covers all environments in that cluster without repeating the file per environment +- **All environments in the repository** - use the global path + +Prefer the **broadest scope that still makes sense** to avoid duplicating files. + +> [!NOTE] +> The file must still be **referenced by name** in each environment's `env_definition.yml` via `envTemplate.envSpecificResourceProfiles`. The location only determines *which file with that name* is actually used when EnvGene finds multiple files with the same name at different levels. + + + +> [!WARNING] +> Placing files with the **same name** at different levels (e.g. a cluster-wide file and an env-specific file with the identical name) is technically supported but makes troubleshooting harder: reading `bss: "prod-cluster-bss"` in `env_definition.yml` gives no indication that a shadow file exists and silently takes priority. **Prefer distinct names** and reference the correct name explicitly - this makes the active override immediately obvious from the inventory alone. + +### 5.2 Create the env-specific override file + +You want all production environments to use 6 replicas. Create a cluster-wide override: + +**`/environments/prod-cluster/resource_profiles/prod-cluster-bss.yml`** + +```yaml +name: "prod-cluster-bss" +baseline: "prod" +description: "Cluster-wide production tuning for BSS" +applications: + - name: "Cloud-BSS" + services: + - name: "bss-processor" + parameters: + - name: "REPLICAS" + value: 6 + - name: "MEMORY_LIMIT" + value: "6Gi" +``` + +### 5.3 Reference the override in `env_definition.yml` + +Update the environment inventory for each environment in `prod-cluster`. Because the file is at cluster level, any environment that references `prod-cluster-bss` will find it automatically via location priority. + +**`/environments/prod-cluster/prod-01/Inventory/env_definition.yml`** + +```yaml +envTemplate: + envSpecificResourceProfiles: + bss: "prod-cluster-bss" +``` + +The key (`bss`) must match the **namespace template name** as defined in the environment template. The value is the filename without the extension. + +By default (`mergeEnvSpecificResourceProfiles: true`) EnvGene **merges** the env-specific file into the template override. The `REPLICAS: 6` and `MEMORY_LIMIT: 6Gi` from `prod-cluster-bss` are applied on top of `prod-bss-override`; all other parameters from the template override are preserved. The resulting Resource Profile Override keeps the name of the Template Override. + +To **replace** the template override entirely instead of merging, set: + +```yaml +inventory: + config: + mergeEnvSpecificResourceProfiles: false +``` + +In this case the resulting Resource Profile Override keeps the name of the env-specific file. + +## Step 6: Override the Override for a Single Environment + +`prod-01` needs to double its CPU limit for an upcoming load test, while all other `prod-cluster` environments keep the cluster-wide profile. + +The recommended approach is to give the env-specific file a **distinct name** and reference it explicitly in `env_definition.yml`. This makes the active override immediately visible in the inventory without having to inspect the filesystem. + +Create the file at the environment-specific path: + +**`/environments/prod-cluster/prod-01/Inventory/resource_profiles/prod-01-bss-loadtest.yml`** + +```yaml +name: "prod-01-bss-loadtest" +baseline: "prod" +description: "Temporary load-test profile for prod-01" +applications: + - name: "Cloud-BSS" + services: + - name: "bss-processor" + parameters: + - name: "REPLICAS" + value: 6 + - name: "CPU_REQUEST" + value: "4000m" + - name: "CPU_LIMIT" + value: "8000m" + - name: "MEMORY_LIMIT" + value: "6Gi" +``` + +Then update `env_definition.yml` for `prod-01` to point to the new file: + +**`/environments/prod-cluster/prod-01/Inventory/env_definition.yml`** + +```yaml +envTemplate: + envSpecificResourceProfiles: + bss: "prod-01-bss-loadtest" +``` + +EnvGene finds `prod-01-bss-loadtest.yml` at the env-specific path and applies it. All other environments in `prod-cluster` still reference `prod-cluster-bss` and are unaffected. + +> [!WARNING] +> An alternative is to create the env-specific file with the **same name** as the cluster-wide file (`prod-cluster-bss.yml`) and skip the `env_definition.yml` change entirely. EnvGene will silently pick up the more specific copy due to location priority. Avoid this pattern: when reading the inventory you cannot tell which file is actually active, which makes incidents significantly harder to diagnose. + +## Step 7: Verify the Result + +After committing your changes, trigger environment generation and inspect two artifacts. + +### 7.1 Profile Override file in `Profiles/` + +The name of the resulting file in `Profiles/` depends on the combination mode: + +| `mergeEnvSpecificResourceProfiles` | Resulting filename in `Profiles/` | +|------------------------------------|---------------------------------------------| +| `true` (default) | Name of the **template** override | +| `false` | Name of the **env-specific** override | + +In this tutorial the default merge mode is used, so the resulting file keeps the template override name: + +```text +environments/prod-cluster/prod-01/Profiles/prod-bss-override.yml +``` + +Open it to verify the merged result. Parameters that came from `prod-01-bss-loadtest` carry an inline `# from prod-01-bss-loadtest` comment, making it clear which env-specific file contributed each value: + +```yaml +applications: +- name: "Cloud-BSS" + services: + - name: "bss-processor" + parameters: + - name: "REPLICAS" + value: 6 # from prod-01-bss-loadtest + - name: "CPU_REQUEST" + value: "4000m" # from prod-01-bss-loadtest + - name: "CPU_LIMIT" + value: "8000m" # from prod-01-bss-loadtest + - name: "MEMORY_REQUEST" + value: "2Gi" + - name: "MEMORY_LIMIT" + value: "6Gi" # from prod-01-bss-loadtest +``` + +`MEMORY_REQUEST` has no comment - it was not present in `prod-01-bss-loadtest` and was kept unchanged from the template override. + +### 7.2 Effective Set + +Then inspect the per-service deployment parameters in the Effective Set: + +```text +environments/prod-cluster/prod-01/effective-set/deployment/bss/Cloud-BSS/values/per-service-parameters/bss-processor/deployment-parameters.yaml +``` + +Each parameter line carries an inline traceability comment showing its ultimate source after baseline + override resolution: + +```yaml +bss-processor: + REPLICAS: 6 #rp-override: prod-bss-override + CPU_REQUEST: 4000m #rp-override: prod-bss-override + CPU_LIMIT: 8000m #rp-override: prod-bss-override + MEMORY_REQUEST: 2Gi #rp-override: prod-bss-override + MEMORY_LIMIT: 6Gi #rp-override: prod-bss-override + HPA_ENABLED: false #rp-baseline: prod +``` + +The two comment formats: + +| Comment | Meaning | +|------------------------|--------------------------------------------------------------------| +| `#rp-baseline: ` | Value comes from the Baseline in the application SBOM | +| `#rp-override: ` | Value was set or overridden by the named Resource Profile Override | + +- **Effective Set comment** - shows which `Profiles/` file contributed the value. +- **`Profiles/` comment** (`# from ...`) - shows which env-specific file overrode that value inside the Profile Override. + +This makes troubleshooting straightforward: if a service is using unexpected resource values, open the Effective Set file, find the parameter, read the comment, and go directly to the source. + +## Summary + +In this tutorial you walked through the full resource profile management workflow: + +- **Baseline** - ships with the application artifact; defines starting performance parameters; read-only from EnvGene's perspective. +- **Template Resource Profile Override** - lives in `/templates/resource_profiles/`; referenced by `profile.name` in Cloud/Namespace templates; applies to all environments using the same template. +- **`template_override`** - replace two near-identical template files with a single base template; assign different profiles in `dev.yml` / `prod.yml` Template Descriptors via the `template_override.profile` block. +- **`overrides-parent`** - when consuming an external versioned parent template artifact, swap or augment its profile in the child Template Descriptor without editing the parent artifact. +- **Environment-Specific Override** - lives in the Instance Repository; referenced via `envSpecificResourceProfiles` in `env_definition.yml`; merged into or replaces the template override depending on `mergeEnvSpecificResourceProfiles`. +- **Override of override** - give the env-specific file a distinct name and reference it explicitly in `env_definition.yml`; use the environment-specific path (`/environments///Inventory/resource_profiles/`) to scope it to one environment, leaving all others unaffected. + +**Related documentation:** + +- [Resource Profile Feature Documentation](/docs/features/resource-profile.md) +- [How to Configure Resource Profiles](/docs/how-to/configure-resource-profiles.md) +- [Template Resource Profile Override Schema](/docs/envgene-objects.md#template-resource-profile-override) +- [Environment Specific Resource Profile Override Schema](/docs/envgene-objects.md#environment-specific-resource-profile-override) +- [Template Override Feature](/docs/features/template-override.md) +- [Template Composition Feature](/docs/features/template-composition.md) +- [Environment Inventory Reference](/docs/envgene-configs.md#env_definitionyml) diff --git a/docs/use-cases/artifact-downloading.md b/docs/use-cases/artifact-downloading.md new file mode 100644 index 000000000..2c2121baf --- /dev/null +++ b/docs/use-cases/artifact-downloading.md @@ -0,0 +1,1359 @@ +# Artifact Downloading Use Cases + +- [Artifact Downloading Use Cases](#artifact-downloading-use-cases) + - [Overview](#overview) + - [Supported Configurations](#supported-configurations) + - [Valid Configuration Combinations for SD/DD Artifacts](#valid-configuration-combinations-for-sddd-artifacts) + - [Valid Configuration Combinations for Environment Template Artifacts](#valid-configuration-combinations-for-environment-template-artifacts) + - [SD/DD Artifact Download](#sddd-artifact-download) + - [UC-AD-SD-1: Download SD from Artifactory with User/Password (AppDef v1 + RegDef v1)](#uc-ad-sd-1-download-sd-from-artifactory-with-userpassword-appdef-v1--regdef-v1) + - [UC-AD-SD-2: Download SD from Artifactory with Anonymous Access (AppDef v1 + RegDef v1)](#uc-ad-sd-2-download-sd-from-artifactory-with-anonymous-access-appdef-v1--regdef-v1) + - [UC-AD-SD-3: Download SD from Nexus with User/Password (AppDef v1 + RegDef v1)](#uc-ad-sd-3-download-sd-from-nexus-with-userpassword-appdef-v1--regdef-v1) + - [UC-AD-SD-4: Download SD from Nexus with Anonymous Access (AppDef v1 + RegDef v1)](#uc-ad-sd-4-download-sd-from-nexus-with-anonymous-access-appdef-v1--regdef-v1) + - [UC-AD-SD-5: Download SD from Artifactory with User/Password (AppDef v1 + RegDef v2)](#uc-ad-sd-5-download-sd-from-artifactory-with-userpassword-appdef-v1--regdef-v2) + - [UC-AD-SD-6: Download SD from Artifactory with Anonymous Access (AppDef v1 + RegDef v2)](#uc-ad-sd-6-download-sd-from-artifactory-with-anonymous-access-appdef-v1--regdef-v2) + - [UC-AD-SD-7: Download SD from Nexus with User/Password (AppDef v1 + RegDef v2)](#uc-ad-sd-7-download-sd-from-nexus-with-userpassword-appdef-v1--regdef-v2) + - [UC-AD-SD-8: Download SD from Nexus with Anonymous Access (AppDef v1 + RegDef v2)](#uc-ad-sd-8-download-sd-from-nexus-with-anonymous-access-appdef-v1--regdef-v2) + - [UC-AD-SD-9: Download SD from AWS CodeArtifact with Secret (AppDef v1 + RegDef v2)](#uc-ad-sd-9-download-sd-from-aws-codeartifact-with-secret-appdef-v1--regdef-v2) + - [UC-AD-SD-10: Download SD from GCP Artifact Registry with Service Account (AppDef v1 + RegDef v2)](#uc-ad-sd-10-download-sd-from-gcp-artifact-registry-with-service-account-appdef-v1--regdef-v2) + - [UC-AD-SD-11: Download Specific Version SD](#uc-ad-sd-11-download-specific-version-sd) + - [Environment Template Artifact Download](#environment-template-artifact-download) + - [UC-AD-ENV-9: Download Template from Artifactory with GAV notation](#uc-ad-env-9-download-template-from-artifactory-with-gav-notation) + - [UC-AD-ENV-10: Download Template from Artifactory with GAV notation and Anonymous Access](#uc-ad-env-10-download-template-from-artifactory-with-gav-notation-and-anonymous-access) + - [UC-AD-ENV-11: Download Template from Nexus with GAV notation](#uc-ad-env-11-download-template-from-nexus-with-gav-notation) + - [UC-SC-NEX-1: Download template artifact from Nexus with custom CA certificate](#uc-sc-nex-1-download-template-artifact-from-nexus-with-custom-ca-certificate) + - [UC-AD-ENV-12: Download Template from Nexus with GAV notation and Anonymous Access](#uc-ad-env-12-download-template-from-nexus-with-gav-notation-and-anonymous-access) + - [UC-AD-ENV-13: Download Template with app ver notation from Artifactory (ArtDef v1)](#uc-ad-env-13-download-template-with-app-ver-notation-from-artifactory-artdef-v1) + - [UC-AD-ENV-14: Download Template with app ver notation from Artifactory and Anonymous Access (ArtDef v1)](#uc-ad-env-14-download-template-with-app-ver-notation-from-artifactory-and-anonymous-access-artdef-v1) + - [UC-AD-ENV-15: Download Template with app ver notation from Nexus (ArtDef v1)](#uc-ad-env-15-download-template-with-app-ver-notation-from-nexus-artdef-v1) + - [UC-AD-ENV-16: Download Template with app ver notation from Nexus and Anonymous Access (ArtDef v1)](#uc-ad-env-16-download-template-with-app-ver-notation-from-nexus-and-anonymous-access-artdef-v1) + - [UC-AD-ENV-17: Download Template from Artifactory with app ver notation (ArtDef v2)](#uc-ad-env-17-download-template-from-artifactory-with-app-ver-notation-artdef-v2) + - [UC-AD-ENV-18: Download Template from Artifactory with app ver notation and Anonymous Access (ArtDef v2)](#uc-ad-env-18-download-template-from-artifactory-with-app-ver-notation-and-anonymous-access-artdef-v2) + - [UC-AD-ENV-19: Download Template from Nexus with app ver notation (ArtDef v2)](#uc-ad-env-19-download-template-from-nexus-with-app-ver-notation-artdef-v2) + - [UC-AD-ENV-20: Download Template from Nexus with app ver notation and Anonymous Access (ArtDef v2)](#uc-ad-env-20-download-template-from-nexus-with-app-ver-notation-and-anonymous-access-artdef-v2) + - [UC-AD-ENV-21: Download Template from AWS CodeArtifact with app ver notation (ArtDef v2)](#uc-ad-env-21-download-template-from-aws-codeartifact-with-app-ver-notation-artdef-v2) + - [UC-AD-ENV-22: Download Template from GCP Artifact Registry with app ver notation (ArtDef v2)](#uc-ad-env-22-download-template-from-gcp-artifact-registry-with-app-ver-notation-artdef-v2) + - [UC-AD-ENV-23: Download SNAPSHOT Template Version](#uc-ad-env-23-download-snapshot-template-version) + - [UC-AD-ENV-24: Download Specific Template Version](#uc-ad-env-24-download-specific-template-version) + - [Error Handling](#error-handling) + - [UC-AD-ERR-1: Handle Missing Application Definition](#uc-ad-err-1-handle-missing-application-definition) + - [UC-AD-ERR-2: Handle Missing Registry Definition](#uc-ad-err-2-handle-missing-registry-definition) + - [UC-AD-ERR-3: Handle Authentication Failure](#uc-ad-err-3-handle-authentication-failure) + - [UC-AD-ERR-4: Handle Missing Artifact Definition](#uc-ad-err-4-handle-missing-artifact-definition) + - [Configuration Examples](#configuration-examples) + - [Registry Definition Examples](#registry-definition-examples) + - [Artifactory / Nexus (RegDef v1.0)](#artifactory--nexus-regdef-v10) + - [Artifactory (RegDef v2.0)](#artifactory-regdef-v20) + - [Nexus (RegDef v2.0)](#nexus-regdef-v20) + - [AWS CodeArtifact (RegDef v2.0)](#aws-codeartifact-regdef-v20) + - [GCP Artifact Registry (RegDef v2.0)](#gcp-artifact-registry-regdef-v20) + - [Artifact Definition Examples](#artifact-definition-examples) + - [Artifact Definition v1.0](#artifact-definition-v10) + - [Artifact Definition v2.0](#artifact-definition-v20) + - [Credentials Configuration Examples](#credentials-configuration-examples) + - [User/Password Authentication](#userpassword-authentication) + - [AWS Secret Authentication](#aws-secret-authentication) + - [GCP Service Account Authentication](#gcp-service-account-authentication) + - [Environment Inventory Examples](#environment-inventory-examples) + - [Template with GAV notation](#template-with-gav-notation) + - [Template with app ver notation](#template-with-app-ver-notation) + +## Overview + +EnvGene downloads three types of artifacts: + +1. **SD (Solution Descriptor)** - processed in `sd_processing` and `effective_set_generation` +2. **DD (Deployment Descriptor)** - processed in `effective_set_generation` +3. **Environment Template** - processed in `app_reg_def_process` + +All artifacts are Maven artifacts and can be stored in various registry types with different authentication methods. + +> [!TIP] +> Complete configuration examples (Registry Definitions, Application Definitions, Artifact Definitions) are available in the [Configuration Examples](#configuration-examples) section at the end of this document. + +## Supported Configurations + +This section provides a quick reference for valid configuration combinations across all supported registry types. + +### Valid Configuration Combinations for SD/DD Artifacts + +| Registry Type | AppDef Version | RegDef Version | Auth Method | Supported | SNAPSHOT | Notes | +|----------------------|----------------|----------------|------------------|------------|-----------|------------------------------------------------------------| +| Artifactory | v1.0 | v1.0 | user_pass | ✅ Yes | ❌ No | Legacy, maintained indefinitely | +| Artifactory | v1.0 | v1.0 | anonymous | ✅ Yes | ❌ No | Legacy, maintained indefinitely | +| Artifactory | v1.0 | v2.0 | user_pass | ✅ Yes | ❌ No | Recommended for new implementations | +| Artifactory | v1.0 | v2.0 | anonymous | ✅ Yes | ❌ No | Recommended for new implementations | +| Nexus | v1.0 | v1.0 | user_pass | ✅ Yes | ❌ No | Legacy, maintained indefinitely | +| Nexus | v1.0 | v1.0 | anonymous | ✅ Yes | ❌ No | Legacy, maintained indefinitely | +| Nexus | v1.0 | v2.0 | user_pass | ✅ Yes | ❌ No | Recommended for new implementations | +| Nexus | v1.0 | v2.0 | anonymous | ✅ Yes | ❌ No | Recommended for new implementations | +| AWS CodeArtifact | v1.0 | v1.0 | any | ❌ No | ❌ No | v1.0 RegDef cannot support AWS auth | +| AWS CodeArtifact | v1.0 | v2.0 | secret | ✅ Yes | ❌ No | Required for AWS. Only secret supported. | +| AWS CodeArtifact | v1.0 | v2.0 | anonymous | ❌ No | ❌ No | Anonymous access not supported for public cloud registries | +| AWS CodeArtifact | v1.0 | v2.0 | assume_role | ❌ No | ❌ No | Not supported | +| GCP GAR | v1.0 | v1.0 | any | ❌ No | ❌ No | v1.0 RegDef cannot support GCP auth | +| GCP GAR | v1.0 | v2.0 | service_account | ✅ Yes | ❌ No | Required for GCP. Only service_account supported. | +| GCP GAR | v1.0 | v2.0 | anonymous | ❌ No | ❌ No | Anonymous access not supported for public cloud registries | +| GCP GAR | v1.0 | v2.0 | federation | ❌ No | ❌ No | Not supported | +| Azure Artifacts | any | any | oauth2 | ❌ No | ❌ No | Azure not supported | + +### Valid Configuration Combinations for Environment Template Artifacts + +| Registry Type | Notation | ArtDef Version | Auth Method | Supported | SNAPSHOT | Notes | +|----------------------|----------|----------------|------------------|------------|-----------|------------------------------------------------------------| +| Artifactory | GAV | N/A (Legacy) | user_pass | ✅ Yes | ✅ Yes | Legacy logic, does NOT use Artifact Definitions | +| Artifactory | GAV | N/A (Legacy) | anonymous | ✅ Yes | ✅ Yes | Legacy logic, does NOT use Artifact Definitions | +| Artifactory | app:ver | v1.0 | user_pass | ✅ Yes | ✅ Yes | Legacy, maintained indefinitely | +| Artifactory | app:ver | v1.0 | anonymous | ✅ Yes | ✅ Yes | Legacy, maintained indefinitely | +| Artifactory | app:ver | v2.0 | user_pass | ✅ Yes | ✅ Yes | Recommended for new implementations | +| Artifactory | app:ver | v2.0 | anonymous | ✅ Yes | ✅ Yes | Recommended for new implementations | +| Nexus | GAV | N/A (Legacy) | user_pass | ✅ Yes | ✅ Yes | Legacy logic, does NOT use Artifact Definitions | +| Nexus | GAV | N/A (Legacy) | anonymous | ✅ Yes | ✅ Yes | Legacy logic, does NOT use Artifact Definitions | +| Nexus | app:ver | v1.0 | user_pass | ✅ Yes | ✅ Yes | Legacy, maintained indefinitely | +| Nexus | app:ver | v1.0 | anonymous | ✅ Yes | ✅ Yes | Legacy, maintained indefinitely | +| Nexus | app:ver | v2.0 | user_pass | ✅ Yes | ✅ Yes | Recommended for new implementations | +| Nexus | app:ver | v2.0 | anonymous | ✅ Yes | ✅ Yes | Recommended for new implementations | +| AWS CodeArtifact | GAV | N/A | any | ❌ No | ❌ No | GAV notation limited to Artifactory/Nexus | +| AWS CodeArtifact | app:ver | v1.0 | any | ❌ No | ❌ No | v1.0 cannot support AWS auth | +| AWS CodeArtifact | app:ver | v2.0 | secret | ✅ Yes | ✅ Yes | Required for AWS. Only secret supported. | +| AWS CodeArtifact | app:ver | v2.0 | anonymous | ❌ No | ❌ No | Anonymous access not supported for public cloud registries | +| AWS CodeArtifact | app:ver | v2.0 | assume_role | ❌ No | ❌ No | Not supported | +| GCP GAR | GAV | N/A | any | ❌ No | ❌ No | GAV notation limited to Artifactory/Nexus | +| GCP GAR | app:ver | v1.0 | any | ❌ No | ❌ No | v1.0 cannot support GCP auth | +| GCP GAR | app:ver | v2.0 | service_account | ✅ Yes | ✅ Yes | Required for GCP. Only service_account supported. | +| GCP GAR | app:ver | v2.0 | anonymous | ❌ No | ❌ No | Anonymous access not supported for public cloud registries | +| GCP GAR | app:ver | v2.0 | federation | ❌ No | ❌ No | Not supported | +| Azure Artifacts | any | any | oauth2 | ❌ No | ❌ No | Azure not supported | + +## SD/DD Artifact Download + +This group covers use cases for downloading Solution Descriptors (SD) and Deployment Descriptors (DD) from various registries. DD artifacts follow the same patterns as SD artifacts. + +### UC-AD-SD-1: Download SD from Artifactory with User/Password (AppDef v1 + RegDef v1) + +**Pre-requisites:** + +1. Application Definition v1.0 exists for SD application +2. Registry Definition v1.0 exists with Artifactory configuration + (See [Artifactory / Nexus RegDef v1.0 example](#artifactory--nexus-regdef-v10)) +3. Credentials stored in `/configuration/credentials/credentials.yml` + (See [User/Password authentication example](#userpassword-authentication)) + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `SD_SOURCE_TYPE: artifact` +3. `SD_VERSION: ` + +**Steps:** + +1. The `process_sd` job runs in the pipeline: + 1. Parses `SD_VERSION` parameter (format: `application:version`) + 2. Resolves application name to Application Definition v1.0 + 3. Extracts `registryName` from AppDef + 4. Resolves registry to Registry Definition v1.0 + 5. Extracts Maven coordinates and Artifactory URL + 6. Authenticates using username/password from credentials + 7. Downloads SD artifact from Artifactory + +**Results:** + +1. SD artifact is downloaded successfully +2. Artifact is available for SD processing in subsequent pipeline jobs + +### UC-AD-SD-2: Download SD from Artifactory with Anonymous Access (AppDef v1 + RegDef v1) + +**Pre-requisites:** + +1. Application Definition v1.0 exists for SD application +2. Registry Definition v1.0 exists with Artifactory configuration +3. Registry Definition **does NOT have** `credentialsId` configured (anonymous access) +4. Artifactory registry allows anonymous/public access + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `SD_SOURCE_TYPE: artifact` +3. `SD_VERSION: ` + +**Steps:** + +1. The `process_sd` job runs in the pipeline: + 1. Parses `SD_VERSION` parameter (format: `application:version`) + 2. Resolves application name to Application Definition v1.0 + 3. Extracts `registryName` from AppDef + 4. Resolves registry to Registry Definition v1.0 + 5. Extracts Maven coordinates and Artifactory URL + 6. Downloads SD artifact from Artifactory without authentication + +**Results:** + +1. SD artifact is downloaded successfully without authentication +2. Artifact is available for SD processing in subsequent pipeline jobs + +### UC-AD-SD-3: Download SD from Nexus with User/Password (AppDef v1 + RegDef v1) + +**Pre-requisites:** + +1. Application Definition v1.0 exists for SD application +2. Registry Definition v1.0 exists with Nexus configuration +3. Credentials stored in `/configuration/credentials/credentials.yml` + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `SD_SOURCE_TYPE: artifact` +3. `SD_VERSION: ` + +**Steps:** + +1. The `process_sd` job runs in the pipeline: + 1. Parses `SD_VERSION` parameter (format: `application:version`) + 2. Resolves application name to Application Definition v1.0 + 3. Extracts `registryName` from AppDef + 4. Resolves registry to Registry Definition v1.0 + 5. Extracts Maven coordinates and Nexus URL + 6. Authenticates using username/password from credentials + 7. Downloads SD artifact from Nexus + +**Results:** + +1. SD artifact is downloaded successfully +2. Artifact is available for SD processing in subsequent pipeline jobs + +### UC-AD-SD-4: Download SD from Nexus with Anonymous Access (AppDef v1 + RegDef v1) + +**Pre-requisites:** + +1. Application Definition v1.0 exists for SD application +2. Registry Definition v1.0 exists with Nexus configuration +3. Registry Definition **does NOT have** `credentialsId` configured (anonymous access) +4. Nexus registry allows anonymous/public access + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `SD_SOURCE_TYPE: artifact` +3. `SD_VERSION: ` + +**Steps:** + +1. The `process_sd` job runs in the pipeline: + 1. Parses `SD_VERSION` parameter (format: `application:version`) + 2. Resolves application name to Application Definition v1.0 + 3. Extracts `registryName` from AppDef + 4. Resolves registry to Registry Definition v1.0 + 5. Extracts Maven coordinates and Nexus URL + 6. Downloads SD artifact from Nexus without authentication + +**Results:** + +1. SD artifact is downloaded successfully without authentication +2. Artifact is available for SD processing in subsequent pipeline jobs + +### UC-AD-SD-5: Download SD from Artifactory with User/Password (AppDef v1 + RegDef v2) + +**Pre-requisites:** + +1. Application Definition v1.0 exists for SD application +2. Registry Definition v2.0 exists with `authConfig` section + (See [Artifactory RegDef v2.0 example](#artifactory-regdef-v20)) +3. Credentials stored in `/configuration/credentials/credentials.yml` + (See [User/Password authentication example](#userpassword-authentication)) + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `SD_SOURCE_TYPE: artifact` +3. `SD_VERSION: ` + +**Steps:** + +1. The `process_sd` job runs in the pipeline: + 1. Parses `SD_VERSION` parameter (format: `application:version`) + 2. Resolves application name to Application Definition v1.0 + 3. Resolves registry to Registry Definition v2.0 + 4. Extracts `mavenConfig.authConfig` reference + 5. Resolves to specific `authConfig` block with `authMethod: user_pass` + 6. Authenticates using username/password from credentials + 7. Downloads SD artifact from Artifactory + +**Results:** + +1. SD artifact is downloaded successfully using RegDef v2.0 enhanced authentication +2. Artifact is available for SD processing in subsequent pipeline jobs + +### UC-AD-SD-6: Download SD from Artifactory with Anonymous Access (AppDef v1 + RegDef v2) + +**Pre-requisites:** + +1. Application Definition v1.0 exists for SD application +2. Registry Definition v2.0 exists with Artifactory configuration +3. Registry Definition **does NOT have** `authConfig` section configured (anonymous access) +4. Artifactory registry allows anonymous/public access + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `SD_SOURCE_TYPE: artifact` +3. `SD_VERSION: ` + +**Steps:** + +1. The `process_sd` job runs in the pipeline: + 1. Parses `SD_VERSION` parameter (format: `application:version`) + 2. Resolves application name to Application Definition v1.0 + 3. Resolves registry to Registry Definition v2.0 + 4. Extracts `mavenConfig` (without authConfig reference) + 5. Downloads SD artifact from Artifactory without authentication + +**Results:** + +1. SD artifact is downloaded successfully without authentication using RegDef v2.0 +2. Artifact is available for SD processing in subsequent pipeline jobs + +### UC-AD-SD-7: Download SD from Nexus with User/Password (AppDef v1 + RegDef v2) + +**Pre-requisites:** + +1. Application Definition v1.0 exists for SD application +2. Registry Definition v2.0 exists with `authConfig` section for Nexus +3. Credentials stored in `/configuration/credentials/credentials.yml` + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `SD_SOURCE_TYPE: artifact` +3. `SD_VERSION: ` + +**Steps:** + +1. The `process_sd` job runs in the pipeline: + 1. Parses `SD_VERSION` parameter (format: `application:version`) + 2. Resolves application name to Application Definition v1.0 + 3. Resolves registry to Registry Definition v2.0 + 4. Extracts `mavenConfig.authConfig` reference + 5. Resolves to specific `authConfig` block with `authMethod: user_pass` + 6. Authenticates using username/password from credentials + 7. Downloads SD artifact from Nexus + +**Results:** + +1. SD artifact is downloaded successfully using RegDef v2.0 enhanced authentication +2. Artifact is available for SD processing in subsequent pipeline jobs + +### UC-AD-SD-8: Download SD from Nexus with Anonymous Access (AppDef v1 + RegDef v2) + +**Pre-requisites:** + +1. Application Definition v1.0 exists for SD application +2. Registry Definition v2.0 exists with Nexus configuration +3. Registry Definition **does NOT have** `authConfig` section configured (anonymous access) +4. Nexus registry allows anonymous/public access + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `SD_SOURCE_TYPE: artifact` +3. `SD_VERSION: ` + +**Steps:** + +1. The `process_sd` job runs in the pipeline: + 1. Parses `SD_VERSION` parameter (format: `application:version`) + 2. Resolves application name to Application Definition v1.0 + 3. Resolves registry to Registry Definition v2.0 + 4. Extracts `mavenConfig` (without authConfig reference) + 5. Downloads SD artifact from Nexus without authentication + +**Results:** + +1. SD artifact is downloaded successfully without authentication using RegDef v2.0 +2. Artifact is available for SD processing in subsequent pipeline jobs + +### UC-AD-SD-9: Download SD from AWS CodeArtifact with Secret (AppDef v1 + RegDef v2) + +**Pre-requisites:** + +1. Application Definition v1.0 exists for SD application +2. Registry Definition v2.0 exists with `provider: aws` and `authMethod: secret` + (See [AWS CodeArtifact RegDef v2.0 example](#aws-codeartifact-regdef-v20)) +3. AWS access key and secret stored in credentials + (See [AWS Secret authentication example](#aws-secret-authentication)) + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `SD_SOURCE_TYPE: artifact` +3. `SD_VERSION: ` + +**Steps:** + +1. The `process_sd` job runs in the pipeline: + 1. Parses `SD_VERSION` parameter (format: `application:version`) + 2. Resolves application name to Application Definition v1.0 + 3. Resolves registry to Registry Definition v2.0 + 4. Extracts AWS configuration (`awsRegion`, `awsDomain`) + 5. Authenticates using AWS access key/secret from credentials + 6. Gets temporary CodeArtifact token + 7. Downloads SD Maven artifact from AWS CodeArtifact + +**Results:** + +1. SD artifact is downloaded successfully from AWS CodeArtifact +2. Artifact is available for SD processing in subsequent pipeline jobs + +### UC-AD-SD-10: Download SD from GCP Artifact Registry with Service Account (AppDef v1 + RegDef v2) + +**Pre-requisites:** + +1. Application Definition v1.0 exists for SD application +2. Registry Definition v2.0 exists with `provider: gcp` and `authMethod: service_account` + (See [GCP Artifact Registry RegDef v2.0 example](#gcp-artifact-registry-regdef-v20)) +3. Service account JSON key stored as credential + (See [GCP Service Account authentication example](#gcp-service-account-authentication)) + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `SD_SOURCE_TYPE: artifact` +3. `SD_VERSION: ` + +**Steps:** + +1. The `process_sd` job runs in the pipeline: + 1. Parses `SD_VERSION` parameter (format: `application:version`) + 2. Resolves application name to Application Definition v1.0 + 3. Resolves registry to Registry Definition v2.0 + 4. Extracts GCP configuration (`gcpProject`, `gcpRegion`) + 5. Loads service account JSON key from credentials + 6. Authenticates to GCP using service account + 7. Downloads SD Maven artifact from GCP Artifact Registry + +**Results:** + +1. SD artifact is downloaded successfully from GCP Artifact Registry +2. Artifact is available for SD processing in subsequent pipeline jobs + +### UC-AD-SD-11: Download Specific Version SD + +**Pre-requisites:** + +1. `SD_VERSION` parameter specifies exact version (e.g., `solution:1.2.3`) +2. Application Definition and Registry Definition are configured + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `SD_SOURCE_TYPE: artifact` +3. `SD_VERSION: ` (specific version, NOT SNAPSHOT) + +**Steps:** + +1. The `process_sd` job runs in the pipeline: + 1. Parses specific version from `SD_VERSION` parameter + 2. Downloads exact version from configured registry + +**Results:** + +1. Specific SD artifact version is downloaded successfully +2. Artifact is available for SD processing in subsequent pipeline jobs + +## Environment Template Artifact Download + +This group covers use cases for downloading Environment Template artifacts from various registries using GAV notation or app:ver notation. + +### UC-AD-ENV-9: Download Template from Artifactory with GAV notation + +**Pre-requisites:** + +1. Environment Inventory exists and specifies template with GAV notation: + + ```yaml + templateArtifact: + registry: "sandbox" + artifact: + group_id: "org.qubership" + artifact_id: "env-template" + version: "1.2.3" + ``` + +2. `registry.yml` exists with Artifactory configuration + (See [Artifactory / Nexus RegDef v1.0 example](#artifactory--nexus-regdef-v10)) +3. Credentials configured for Artifactory + (See [User/Password authentication example](#userpassword-authentication)) + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `app_reg_def_process` job runs in the pipeline: + 1. Reads Environment Inventory from `/environments///Inventory/env_definition.yml` + 2. Parses GAV coordinates from `templateArtifact` section + 3. Resolves registry from `registry.yml` + 4. Authenticates using credentials + 5. Downloads template artifact from Artifactory using Maven coordinates + +**Results:** + +1. Template artifact is downloaded successfully +2. Template is available for Environment Instance generation + +### UC-AD-ENV-10: Download Template from Artifactory with GAV notation and Anonymous Access + +**Pre-requisites:** + +1. Environment Inventory exists and specifies template with GAV notation: + + ```yaml + templateArtifact: + registry: "sandbox" + artifact: + group_id: "org.qubership" + artifact_id: "env-template" + version: "1.2.3" + ``` + +2. `registry.yml` exists with Artifactory configuration +3. `registry.yml` **does NOT have** `credentialsId` configured (anonymous access) +4. Artifactory registry allows anonymous/public access + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `app_reg_def_process` job runs in the pipeline: + 1. Reads Environment Inventory + 2. Parses GAV coordinates from `templateArtifact` section + 3. Resolves registry from `registry.yml` + 4. Downloads template artifact from Artifactory without authentication + +**Results:** + +1. Template artifact is downloaded successfully without authentication +2. Template is available for Environment Instance generation + +### UC-AD-ENV-11: Download Template from Nexus with GAV notation + +**Pre-requisites:** + +1. Environment Inventory exists and specifies template with GAV notation (similar to UC-AD-ENV-9, but with Nexus registry) +2. `registry.yml` exists with Nexus configuration +3. Credentials configured for Nexus + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `app_reg_def_process` job runs in the pipeline: + 1. Reads Environment Inventory + 2. Parses GAV coordinates from `templateArtifact` section + 3. Resolves registry from `registry.yml` + 4. Authenticates using credentials + 5. Downloads template artifact from Nexus + +**Results:** + +1. Template artifact is downloaded successfully +2. Template is available for Environment Instance generation + +### UC-SC-NEX-1: Download template artifact from Nexus with custom CA certificate + +**Pre-requisites:** + +1. Template artifact is uploaded to Nexus and available for download. +2. Environment Inventory exists and specifies template with GAV notation. +3. `registry.yml` exists with Nexus configuration and credentials. +4. Nexus endpoint uses certificate chain signed by private or internal CA. +5. Instance repository contains CA certificate chain file in `configuration/certs/`. + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `app_reg_def_process` job runs in the pipeline: + 1. Reads Environment Inventory. + 2. Parses GAV coordinates from `templateArtifact` section. + 3. Resolves registry from `registry.yml`. + 4. Loads CA certificates from `configuration/certs/` into runner trust. + 5. Authenticates using credentials. + 6. Connects to Nexus over TLS and downloads template artifact. + +**Results:** + +1. TLS connection to Nexus is established successfully. +2. Template artifact is downloaded successfully. +3. No `CERTIFICATE_VERIFY_FAILED` or trust errors appear in logs. + +### UC-AD-ENV-12: Download Template from Nexus with GAV notation and Anonymous Access + +**Pre-requisites:** + +1. Environment Inventory exists and specifies template with GAV notation (similar to UC-AD-ENV-9) +2. `registry.yml` exists with Nexus configuration +3. `registry.yml` **does NOT have** `credentialsId` configured (anonymous access) +4. Nexus registry allows anonymous/public access + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `app_reg_def_process` job runs in the pipeline: + 1. Reads Environment Inventory + 2. Parses GAV coordinates from `templateArtifact` section + 3. Resolves registry from `registry.yml` + 4. Downloads template artifact from Nexus without authentication + +**Results:** + +1. Template artifact is downloaded successfully without authentication +2. Template is available for Environment Instance generation + +### UC-AD-ENV-13: Download Template with app ver notation from Artifactory (ArtDef v1) + +**Pre-requisites:** + +1. Environment Inventory exists and specifies template with `application:version` notation: + + ```yaml + envTemplate: + artifact: "env-template:1.2.3" + ``` + +2. Artifact Definition v1.0 exists at `/configuration/artifact_definitions/env-template.yaml` +3. Registry configuration in Artifact Definition points to Artifactory +4. Credentials configured + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `app_reg_def_process` job runs in the pipeline: + 1. Reads Environment Inventory + 2. Parses `application:version` from `envTemplate.artifact` + 3. Resolves to Artifact Definition v1.0 + 4. Extracts Maven GAV coordinates and registry information + 5. Authenticates using credentials + 6. Downloads template artifact from Artifactory + +**Results:** + +1. Template artifact is downloaded successfully using ArtDef v1.0 +2. Template is available for Environment Instance generation + +### UC-AD-ENV-14: Download Template with app ver notation from Artifactory and Anonymous Access (ArtDef v1) + +**Pre-requisites:** + +1. Environment Inventory exists and specifies template with `application:version` notation: + + ```yaml + envTemplate: + artifact: "env-template:1.2.3" + ``` + +2. Artifact Definition v1.0 exists at `/configuration/artifact_definitions/env-template.yaml` +3. Registry configuration in Artifact Definition **does NOT have** `credentialsId` (anonymous access) +4. Artifactory registry allows anonymous/public access + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `app_reg_def_process` job runs in the pipeline: + 1. Reads Environment Inventory + 2. Parses `application:version` from `envTemplate.artifact` + 3. Resolves to Artifact Definition v1.0 + 4. Extracts Maven GAV coordinates and registry information + 5. Downloads template artifact from Artifactory without authentication + +**Results:** + +1. Template artifact is downloaded successfully without authentication using ArtDef v1.0 +2. Template is available for Environment Instance generation + +### UC-AD-ENV-15: Download Template with app ver notation from Nexus (ArtDef v1) + +**Pre-requisites:** + +1. Environment Inventory exists and specifies template with `application:version` notation: + + ```yaml + envTemplate: + artifact: "env-template:1.2.3" + ``` + +2. Artifact Definition v1.0 exists at `/configuration/artifact_definitions/env-template.yaml` + (See [Artifact Definition v1.0 example](#artifact-definition-v10)) +3. Registry configuration in Artifact Definition points to Nexus + (See [Artifactory / Nexus RegDef v1.0 example](#artifactory--nexus-regdef-v10)) +4. Credentials configured + (See [User/Password authentication example](#userpassword-authentication)) + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `app_reg_def_process` job runs in the pipeline: + 1. Reads Environment Inventory + 2. Parses `application:version` from `envTemplate.artifact` + 3. Resolves to Artifact Definition v1.0 + 4. Extracts Maven GAV coordinates and registry information + 5. Authenticates using credentials + 6. Downloads template artifact from Nexus + +**Results:** + +1. Template artifact is downloaded successfully using ArtDef v1.0 +2. Template is available for Environment Instance generation + +### UC-AD-ENV-16: Download Template with app ver notation from Nexus and Anonymous Access (ArtDef v1) + +**Pre-requisites:** + +1. Environment Inventory exists and specifies template with `application:version` notation (similar to UC-AD-ENV-15) +2. Artifact Definition v1.0 exists with Nexus registry configuration +3. Registry configuration **does NOT have** `credentialsId` (anonymous access) +4. Nexus registry allows anonymous/public access + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `app_reg_def_process` job runs in the pipeline: + 1. Reads Environment Inventory + 2. Parses `application:version` from `envTemplate.artifact` + 3. Resolves to Artifact Definition v1.0 + 4. Extracts Maven GAV coordinates and registry information + 5. Downloads template artifact from Nexus without authentication + +**Results:** + +1. Template artifact is downloaded successfully without authentication using ArtDef v1.0 +2. Template is available for Environment Instance generation + +### UC-AD-ENV-17: Download Template from Artifactory with app ver notation (ArtDef v2) + +**Pre-requisites:** + +1. Environment Inventory specifies template with `application:version` format + (See [Template with app:ver notation example](#template-with-app-ver-notation)) +2. Artifact Definition v2.0 exists with `authConfig` section + (See [Artifact Definition v2.0 example](#artifact-definition-v20)) +3. `authConfig` specifies `authMethod: user_pass` +4. Credentials configured + (See [User/Password authentication example](#userpassword-authentication)) + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `app_reg_def_process` job runs in the pipeline: + 1. Reads Environment Inventory + 2. Parses `application:version` from `envTemplate.artifact` + 3. Resolves to Artifact Definition v2.0 + 4. Extracts `authConfig` reference + 5. Authenticates using username/password from credentials + 6. Downloads template artifact from Artifactory + +**Results:** + +1. Template artifact is downloaded successfully using ArtDef v2.0 with enhanced authentication +2. Template is available for Environment Instance generation + +### UC-AD-ENV-18: Download Template from Artifactory with app ver notation and Anonymous Access (ArtDef v2) + +**Pre-requisites:** + +1. Environment Inventory specifies template with `application:version` format +2. Artifact Definition v2.0 exists +3. **No** `authConfig` section in ArtDef v2.0 (anonymous access) +4. Artifactory registry allows anonymous/public access + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `app_reg_def_process` job runs in the pipeline: + 1. Reads Environment Inventory + 2. Parses `application:version` from `envTemplate.artifact` + 3. Resolves to Artifact Definition v2.0 + 4. Extracts `mavenConfig` (without authConfig reference) + 5. Downloads template artifact from Artifactory without authentication + +**Results:** + +1. Template artifact is downloaded successfully without authentication using ArtDef v2.0 +2. Template is available for Environment Instance generation + +### UC-AD-ENV-19: Download Template from Nexus with app ver notation (ArtDef v2) + +**Pre-requisites:** + +1. Environment Inventory specifies template with `application:version` format +2. Artifact Definition v2.0 exists for Nexus +3. `authConfig` specifies `authMethod: user_pass` +4. Credentials configured + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `app_reg_def_process` job runs in the pipeline: + 1. Reads Environment Inventory + 2. Parses `application:version` from `envTemplate.artifact` + 3. Resolves to Artifact Definition v2.0 + 4. Extracts `authConfig` reference + 5. Authenticates using username/password from credentials + 6. Downloads template artifact from Nexus + +**Results:** + +1. Template artifact is downloaded successfully using ArtDef v2.0 with enhanced authentication +2. Template is available for Environment Instance generation + +### UC-AD-ENV-20: Download Template from Nexus with app ver notation and Anonymous Access (ArtDef v2) + +**Pre-requisites:** + +1. Environment Inventory specifies template with `application:version` format +2. Artifact Definition v2.0 exists for Nexus +3. **No** `authConfig` section in ArtDef v2.0 (anonymous access) +4. Nexus registry allows anonymous/public access + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `app_reg_def_process` job runs in the pipeline: + 1. Reads Environment Inventory + 2. Parses `application:version` from `envTemplate.artifact` + 3. Resolves to Artifact Definition v2.0 + 4. Extracts `mavenConfig` (without authConfig reference) + 5. Downloads template artifact from Nexus without authentication + +**Results:** + +1. Template artifact is downloaded successfully without authentication using ArtDef v2.0 +2. Template is available for Environment Instance generation + +### UC-AD-ENV-21: Download Template from AWS CodeArtifact with app ver notation (ArtDef v2) + +**Pre-requisites:** + +1. Environment Inventory specifies template with `application:version` format +2. Artifact Definition v2.0 exists with `provider: aws` and `authMethod: secret` +3. AWS credentials configured (access key + secret) + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `app_reg_def_process` job runs in the pipeline: + 1. Reads Environment Inventory + 2. Parses `application:version` from `envTemplate.artifact` + 3. Resolves to Artifact Definition v2.0 + 4. Authenticates to AWS using access key/secret from credentials + 5. Gets temporary CodeArtifact token + 6. Downloads template Maven artifact from AWS CodeArtifact + +**Results:** + +1. Template artifact is downloaded successfully from AWS CodeArtifact +2. Template is available for Environment Instance generation + +### UC-AD-ENV-22: Download Template from GCP Artifact Registry with app ver notation (ArtDef v2) + +**Pre-requisites:** + +1. Environment Inventory specifies template with `application:version` format +2. Artifact Definition v2.0 exists with `provider: gcp` and `authMethod: service_account` +3. GCP service account JSON key configured + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `app_reg_def_process` job runs in the pipeline: + 1. Reads Environment Inventory + 2. Parses `application:version` from `envTemplate.artifact` + 3. Resolves to Artifact Definition v2.0 + 4. Loads service account JSON key + 5. Authenticates to GCP using service account + 6. Downloads template Maven artifact from GCP Artifact Registry + +**Results:** + +1. Template artifact is downloaded successfully from GCP Artifact Registry +2. Template is available for Environment Instance generation + +### UC-AD-ENV-23: Download SNAPSHOT Template Version + +**Pre-requisites:** + +1. Environment Inventory specifies template with SNAPSHOT version +2. Registry supports SNAPSHOT resolution + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `app_reg_def_process` job runs in the pipeline: + 1. Reads Environment Inventory + 2. Detects SNAPSHOT version in template specification + 3. Resolves SNAPSHOT to latest available version in registry + 4. Downloads latest artifact version + +**Results:** + +1. Latest SNAPSHOT template artifact is downloaded successfully +2. Template is available for Environment Instance generation + +### UC-AD-ENV-24: Download Specific Template Version + +**Pre-requisites:** + +1. Environment Inventory specifies template with exact version (not SNAPSHOT) + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `app_reg_def_process` job runs in the pipeline: + 1. Reads Environment Inventory + 2. Parses exact version from template specification + 3. Downloads specified version from registry + +**Results:** + +1. Specific template artifact version is downloaded successfully +2. Template is available for Environment Instance generation + +## Error Handling + +This group covers error scenarios that can occur during artifact download operations. + +### UC-AD-ERR-1: Handle Missing Application Definition + +**Pre-requisites:** + +1. Pipeline parameter specifies `SD_VERSION` or `DD_VERSION` with `application:version` format +2. Application Definition for specified application does NOT exist + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters that trigger artifact download (e.g., `SD_VERSION: `) + +**Steps:** + +1. Artifact download process attempts to resolve Application Definition +2. Definition file is not found at expected location +3. Pipeline job fails with clear error message indicating missing AppDef + +**Results:** + +1. Pipeline execution fails +2. Error message clearly indicates which Application Definition is missing +3. Error message includes expected file path + +### UC-AD-ERR-2: Handle Missing Registry Definition + +**Pre-requisites:** + +1. Application Definition or Artifact Definition references a registry +2. Registry Definition for specified registry does NOT exist + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters that trigger artifact download + +**Steps:** + +1. Artifact download process attempts to resolve Registry Definition +2. Definition file is not found at expected location +3. Pipeline job fails with clear error message indicating missing RegDef + +**Results:** + +1. Pipeline execution fails +2. Error message clearly indicates which Registry Definition is missing +3. Error message includes expected file path and registry name + +### UC-AD-ERR-3: Handle Authentication Failure + +**Pre-requisites:** + +1. Artifact download requires authentication +2. Authentication credentials are invalid, expired, or missing + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters that trigger artifact download from authenticated registry + +**Steps:** + +1. Artifact download process attempts authentication +2. Authentication fails (invalid credentials, network issue, or authorization problem) +3. Pipeline implements retry logic for transient failures +4. If retries exhausted, pipeline job fails with clear error message + +**Results:** + +1. For transient failures: Retry mechanism attempts to recover +2. For permanent failures: Pipeline execution fails with clear error message +3. Error message indicates authentication failure and suggests troubleshooting steps + +### UC-AD-ERR-4: Handle Missing Artifact Definition + +**Pre-requisites:** + +1. Environment Inventory specifies template with `application:version` notation +2. Artifact Definition for specified application does NOT exist + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `ENV_BUILDER: true` + +**Steps:** + +1. Template download process attempts to resolve Artifact Definition +2. Definition file is not found at expected location +3. Pipeline job fails with clear error message indicating missing ArtDef + +**Results:** + +1. Pipeline execution fails +2. Error message clearly indicates which Artifact Definition is missing +3. Error message includes expected file path + +## Configuration Examples + +This section provides complete configuration examples for definitions referenced in the use cases above. + +### Registry Definition Examples + +#### Artifactory / Nexus (RegDef v1.0) + +> [!NOTE] +> The structure is identical for both Artifactory and Nexus. Only the values differ (registry names, URLs, repository names). +> +> RegDef v1.0 does NOT have a `version` field - the absence of this field indicates v1.0. +> +> This example shows only `mavenConfig` section relevant to Maven artifact download use cases. Full Registry Definition v1.0 requires additional sections (`dockerConfig`, `helmConfig`, etc.) - see [Registry Definition schema](/docs/envgene-objects.md#registry-definition-v10) for complete structure. + +```yaml +name: "artifactory-maven" +credentialsId: "artifactory-creds" +mavenConfig: + repositoryDomainName: "artifactory.example.com" + fullRepositoryUrl: "https://artifactory.example.com/artifactory" + targetSnapshot: "libs-snapshot-local" + targetStaging: "libs-staging-local" + targetRelease: "libs-release-local" + snapshotGroup: "libs-snapshot" + releaseGroup: "libs-release" +dockerConfig: + snapshotUri: "" + stagingUri: "" + releaseUri: "" + groupUri: "" + snapshotRepoName: "" + stagingRepoName: "" + releaseRepoName: "" + groupName: "" +``` + +#### Artifactory (RegDef v2.0) + +> [!NOTE] +> For `provider: nexus` or `provider: artifactory`, the `repositoryDomainName` field contains a full URL. +> For cloud providers (`aws`, `gcp`, `azure`), it also contains a full URL. +> This is a known gap in the model naming convention and will be addressed in future versions. + +```yaml +version: "2.0" +name: "artifactory-maven" +authConfig: + artifactory-auth: + provider: "artifactory" + authMethod: "user_pass" + credentialsId: "artifactory-creds" +mavenConfig: + authConfig: "artifactory-auth" + repositoryDomainName: "https://artifactory.example.com/artifactory" + targetSnapshot: "libs-snapshot-local" + targetStaging: "libs-staging-local" + targetRelease: "libs-release-local" + snapshotGroup: "libs-snapshot" + releaseGroup: "libs-release" +``` + +#### Nexus (RegDef v2.0) + +> [!NOTE] +> For `provider: nexus` or `provider: artifactory`, the `repositoryDomainName` field contains a full URL. +> For cloud providers (`aws`, `gcp`, `azure`), it also contains a full URL. +> This is a known gap in the model naming convention and will be addressed in future versions. + +```yaml +version: "2.0" +name: "nexus-maven" +authConfig: + nexus-auth: + provider: "nexus" + authMethod: "user_pass" + credentialsId: "nexus-creds" +mavenConfig: + authConfig: "nexus-auth" + repositoryDomainName: "https://nexus.example.com/repository" + targetSnapshot: "maven-snapshots" + targetStaging: "maven-staging" + targetRelease: "maven-releases" + snapshotGroup: "maven-public-snapshots" + releaseGroup: "maven-public" +``` + +#### AWS CodeArtifact (RegDef v2.0) + +> [!NOTE] +> For `provider: nexus` or `provider: artifactory`, the `repositoryDomainName` field contains a full URL. +> For cloud providers (`aws`, `gcp`, `azure`), it also contains a full URL. +> This is a known gap in the model naming convention and will be addressed in future versions. + +```yaml +version: "2.0" +name: "aws-codeartifact" +authConfig: + aws-auth: + provider: "aws" + authMethod: "secret" + credentialsId: "aws-secret-creds" + awsRegion: "us-east-1" + awsDomain: "my-domain" +mavenConfig: + authConfig: "aws-auth" + repositoryDomainName: "https://my-domain-123456789012.d.codeartifact.us-east-1.amazonaws.com/maven/maven-central-store" +``` + +#### GCP Artifact Registry (RegDef v2.0) + +> [!NOTE] +> For `provider: nexus` or `provider: artifactory`, the `repositoryDomainName` field contains a full URL. +> For cloud providers (`aws`, `gcp`, `azure`), it also contains a full URL. +> This is a known gap in the model naming convention and will be addressed in future versions. + +```yaml +version: "2.0" +name: "gcp-artifact-registry" +authConfig: + gcp-auth: + provider: "gcp" + authMethod: "service_account" + credentialsId: "gcp-sa-key" + gcpRegion: "us-central1" +mavenConfig: + authConfig: "gcp-auth" + repositoryDomainName: "https://us-central1-maven.pkg.dev/my-project/maven-repo" +``` + +### Artifact Definition Examples + +#### Artifact Definition v1.0 + +```yaml +name: "env-template" +groupId: "com.example.templates" +artifactId: "env-template" +registry: + name: "artifactory-maven" + credentialsId: "artifactory-creds" + mavenConfig: + repositoryDomainName: "https://artifactory.example.com" + targetSnapshot: "libs-snapshot-local" + targetStaging: "libs-staging-local" + targetRelease: "libs-release-local" +``` + +#### Artifact Definition v2.0 + +```yaml +version: "2.0" +name: "env-template" +groupId: "com.example.templates" +artifactId: "env-template" +registry: + name: "artifactory-maven" + authConfig: + artifactory-auth: + provider: "artifactory" + authMethod: "user_pass" + credentialsId: "artifactory-creds" + mavenConfig: + authConfig: "artifactory-auth" + repositoryDomainName: "https://artifactory.example.com" + targetSnapshot: "libs-snapshot-local" + targetStaging: "libs-staging-local" + targetRelease: "libs-release-local" + snapshotGroup: "libs-snapshot" + releaseGroup: "libs-release" +``` + +### Credentials Configuration Examples + +#### User/Password Authentication + +```yaml +artifactory-auth: + type: usernamePassword + data: + username: "user-placeholder-123" + password: "pass-placeholder-123" +``` + +#### AWS Secret Authentication + +```yaml +aws-secret-creds: + type: secret + data: + secret: TBD +``` + +#### GCP Service Account Authentication + +```yaml +gcp-sa-key: + type: secret + data: + secret: TBD +``` + +### Environment Inventory Examples + +#### Template with GAV notation + +```yaml +templateArtifact: + registry: "artifactory-maven" + artifact: + group_id: "com.example.templates" + artifact_id: "env-template" + version: "1.2.3" +``` + +#### Template with app ver notation + +```yaml +envTemplate: + artifact: "env-template:1.2.3" +``` diff --git a/docs/use-cases/auto-environment-name.md b/docs/use-cases/auto-environment-name.md new file mode 100644 index 000000000..63d88d355 --- /dev/null +++ b/docs/use-cases/auto-environment-name.md @@ -0,0 +1,206 @@ +# Automatic Environment Name Derivation Use Cases + +- [Automatic Environment Name Derivation Use Cases](#automatic-environment-name-derivation-use-cases) + - [Overview](#overview) + - [Environment Name Derivation Scenarios](#environment-name-derivation-scenarios) + - [UC-AEN-END-1: Environment with no explicit environmentName defined](#uc-aen-end-1-environment-with-no-explicit-environmentname-defined) + - [UC-AEN-END-2: Environment with explicit environmentName defined](#uc-aen-end-2-environment-with-explicit-environmentname-defined) + - [UC-AEN-END-3: Environment with explicit environmentName different from folder name](#uc-aen-end-3-environment-with-explicit-environmentname-different-from-folder-name) + - [UC-AEN-END-4: Invalid folder structure for environment](#uc-aen-end-4-invalid-folder-structure-for-environment) + - [UC-AEN-END-5: Template rendering with derived environment name](#uc-aen-end-5-template-rendering-with-derived-environment-name) + +## Overview + +This document describes use cases for automatic environment name derivation. + +Feature reference: [Automatic Environment Name Derivation](/docs/features/auto-env-name-derivation.md). + +## Environment Name Derivation Scenarios + +### UC-AEN-END-1: Environment with no explicit environmentName defined + +**Pre-requisites:** + +1. Environment definition file exists at path `/environments///Inventory/env_definition.yml`. +2. The `environmentName` attribute is not defined in `env_definition.yml`: + + ```yaml + inventory: + # environmentName is not defined + tenantName: "Applications" + cloudName: "cluster01" + envTemplate: + name: "simple" + artifact: "project-env-template:master_20231024-080204" + ``` + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: cluster01/env01` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `env_build` job runs in the pipeline. +2. EnvGene reads environment path from `ENV_NAMES`. +3. EnvGene loads `env_definition.yml` and determines environment name. + +**Results:** + +1. The environment is successfully created. +2. The environment name is derived from the folder name `env01`. +3. The environment context contains `current_env.name` with the value `env01`. +4. The generated environment instance references the correct environment name. + +### UC-AEN-END-2: Environment with explicit environmentName defined + +**Pre-requisites:** + +1. Environment definition file exists at path `/environments///Inventory/env_definition.yml`. +2. The `environmentName` attribute is explicitly defined in `env_definition.yml`: + + ```yaml + inventory: + environmentName: "env02" + tenantName: "Applications" + cloudName: "cluster01" + envTemplate: + name: "simple" + artifact: "project-env-template:master_20231024-080204" + ``` + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: cluster01/env02` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `env_build` job runs in the pipeline. +2. EnvGene reads environment path from `ENV_NAMES`. +3. EnvGene loads `env_definition.yml` and uses explicit `environmentName`. + +**Results:** + +1. The environment is successfully created. +2. The explicitly defined environment name `env02` is used. +3. The environment context contains `current_env.name` with the value `env02`. +4. The generated environment instance references the correct environment name. + +### UC-AEN-END-3: Environment with explicit environmentName different from folder name + +**Pre-requisites:** + +1. Environment definition file exists at path `/environments///Inventory/env_definition.yml`. +2. The `environmentName` attribute is explicitly defined with a value different from folder name: + + ```yaml + inventory: + environmentName: "custom-env" + tenantName: "Applications" + cloudName: "cluster01" + envTemplate: + name: "simple" + artifact: "project-env-template:master_20231024-080204" + ``` + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: cluster01/env03` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `env_build` job runs in the pipeline. +2. EnvGene reads environment path from `ENV_NAMES`. +3. EnvGene loads `env_definition.yml` and uses explicit `environmentName`. + +**Results:** + +1. The environment is successfully created. +2. The explicitly defined environment name `custom-env` is used instead of folder name. +3. The environment context contains `current_env.name` with the value `custom-env`. +4. The generated environment instance references the correct environment name. + +### UC-AEN-END-4: Invalid folder structure for environment + +**Pre-requisites:** + +1. Environment definition file exists at invalid path that does not follow expected structure. +2. The `environmentName` attribute is not defined in `env_definition.yml`: + + ```yaml + inventory: + # environmentName is not defined + tenantName: "Applications" + cloudName: "cluster01" + envTemplate: + name: "simple" + artifact: "project-env-template:master_20231024-080204" + ``` + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: invalid-structure` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `env_build` job runs in the pipeline. +2. EnvGene attempts to determine environment name from path. +3. EnvGene detects invalid folder structure. + +**Results:** + +1. Environment creation fails with an appropriate error message. +2. The error message indicates that environment name could not be determined from path. +3. The error message suggests checking folder structure. + +### UC-AEN-END-5: Template rendering with derived environment name + +**Pre-requisites:** + +1. Environment definition file exists at path `/environments///Inventory/env_definition.yml`. +2. The `environmentName` attribute is not defined in `env_definition.yml`. +3. Environment template contains references to `current_env.name`: + + ```yaml + # cloud.yml.j2 + name: "{{ current_env.name }}-cloud" + description: "Cloud for {{ current_env.name }} environment" + ``` + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: cluster01/env04` +2. `ENV_BUILDER: true` + +**Steps:** + +1. The `env_build` job runs in the pipeline. +2. EnvGene derives environment name from folder path. +3. EnvGene renders template with `current_env.name`. + +**Results:** + +1. The environment is successfully created. +2. The environment name is derived from folder name `env04`. +3. The template is rendered with derived environment name: + + ```yaml + # Rendered cloud.yml + name: "env04-cloud" + description: "Cloud for env04 environment" + ``` + +4. All template variables referencing `current_env.name` are substituted with derived name. diff --git a/docs/use-cases/calculator-cli.md b/docs/use-cases/calculator-cli.md index 1b71765d6..1a67e74fb 100644 --- a/docs/use-cases/calculator-cli.md +++ b/docs/use-cases/calculator-cli.md @@ -178,7 +178,7 @@ Instance pipeline (GitLab or GitHub) is started with parameters: ## Parameter Type Preservation in Macro Resolution -This section covers use cases for [Macro Parameter Resolution](/docs/features/calculator-cli.md#version-20-macros) in Effective Set v2.0. The Calculator CLI resolves parameter references while preserving the original parameter types according to [Parameter type conversion](/docs/features/calculator-cli.md#version-20-parameter-type-conversion) rules. +This section covers use cases for [Macro Parameter Resolution](/docs/template-macros.md#calculator-command-line-tool-macros) in Effective Set v2.0. The Calculator CLI resolves parameter references while preserving the original parameter types according to [Parameter type conversion](/docs/features/calculator-cli.md#version-20-parameter-type-conversion) rules. ### UC-CC-MR-1: Simple Type Resolution diff --git a/docs/use-cases/credential-rotation.md b/docs/use-cases/credential-rotation.md new file mode 100644 index 000000000..29a2f487e --- /dev/null +++ b/docs/use-cases/credential-rotation.md @@ -0,0 +1,441 @@ +# Credential Rotation Use Cases + +- [Credential Rotation Use Cases](#credential-rotation-use-cases) + - [Overview](#overview) + - [UC-CR-TPR-1: Update Credential from Pipeline Parameter](#uc-cr-tpr-1-update-credential-from-pipeline-parameter) + - [UC-CR-TPR-2: Update Credential from Deployment Parameter](#uc-cr-tpr-2-update-credential-from-deployment-parameter) + - [UC-CR-TPR-3: Update Credentials from Multiple rotation_items](#uc-cr-tpr-3-update-credentials-from-multiple-rotation_items) + - [Affected Credential Handling](#affected-credential-handling) + - [UC-CR-LCH-1: Reject Affected Credential Update](#uc-cr-lch-1-reject-affected-credential-update) + - [UC-CR-LCH-2: Update Affected Credentials in Force Mode](#uc-cr-lch-2-update-affected-credentials-in-force-mode) + - [UC-CR-VAL-1: Fail When No Affected Parameters Found](#uc-cr-val-1-fail-when-no-affected-parameters-found) + - [Encryption Processing](#encryption-processing) + - [Successful Update with Encryption Enabled](#successful-update-with-encryption-enabled) + - [UC-CR-ENC-1: Update Credentials with Plaintext Payload when Encryption Is Enabled](#uc-cr-enc-1-update-credentials-with-plaintext-payload-when-encryption-is-enabled) + - [UC-CR-ENC-2: Update Credentials with Encrypted Payload when Encryption Is Enabled](#uc-cr-enc-2-update-credentials-with-encrypted-payload-when-encryption-is-enabled) + - [Successful Update with Encryption Disabled](#successful-update-with-encryption-disabled) + +## Overview + +This document contains use cases for [Credential Rotation](/docs/features/cred-rotation.md). + +It describes parameter targeting, affected-credentials handling with force mode, and encryption processing for `credential_rotation`. + +### Successful Update without Affected Credentials + +This group covers successful rotation when the target credential is not linked to other parameters. + +### UC-CR-TPR-1: Update Credential from Pipeline Parameter + +**Pre-requisites:** + +1. `env_inventory_generation_job` must be launched in the pipeline run. +2. Environment Instance contains a Namespace with `name` matching ``. +3. The Namespace contains a sensitive parameter in `e2eParameters` linked via the `cred` macro. +4. The referenced Credential exists in the Environment Credentials file or in a Shared Credentials file. +5. No other parameters reference the same `cred-id` and credential field. +6. `CRED_ROTATION_PAYLOAD` contains a valid `rotation_items` entry for Namespace-level pipeline context: + + ```json + { + "rotation_items": [ + { + "namespace": "", + "context": "pipeline", + "parameter_key": "", + "parameter_value": "" + } + ] + } + ``` + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: /` +2. `CRED_ROTATION_PAYLOAD: ` +3. `CRED_ROTATION_FORCE: false` + +**Steps:** + +1. The `credential_rotation` job runs in the pipeline: + 1. Finds the target Namespace from the payload. + 2. Uses the `pipeline` context to look for the parameter in `e2eParameters`. + 3. Determines which credential field is linked to the target parameter. + 4. Searches for affected credentials linked to the same `cred-id` and credential field. + 5. Finds no affected credentials and continues the flow. + 6. Updates credential value for the target parameter. + +**Results:** + +1. The credential value is updated successfully. +2. The job completes with success status. + +### UC-CR-TPR-2: Update Credential from Deployment Parameter + +**Pre-requisites:** + +1. `env_inventory_generation_job` must be launched in the pipeline run. +2. Environment Instance contains a Namespace with `name` matching ``. +3. That Namespace contains an Application with `name` matching ``. +4. The Application contains a sensitive parameter in `deployParameters`. +5. The parameter is linked via the `cred` macro to an existing credential. +6. No other parameters reference the same `cred-id` and credential field. +7. `CRED_ROTATION_PAYLOAD` contains a valid `rotation_items` entry for Application-level deployment context: + + ```json + { + "rotation_items": [ + { + "namespace": "", + "application": "", + "context": "deployment", + "parameter_key": "db.connection.password", + "parameter_value": "" + } + ] + } + ``` + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: /` +2. `CRED_ROTATION_PAYLOAD: ` +3. `CRED_ROTATION_FORCE: false` + +**Steps:** + +1. The `credential_rotation` job runs in the pipeline: + 1. Finds the target Namespace and the Application specified in the payload. + 2. Uses the `deployment` context to look for the parameter in `deployParameters`. + 3. Determines which credential field is linked to the target parameter. + 4. Searches for affected credentials linked to the same `cred-id` and credential field. + 5. Finds no affected credentials and continues the flow. + 6. Updates credential value for the target parameter. + +**Results:** + +1. The credential value is updated successfully. +2. The job completes with success status. + +### UC-CR-TPR-3: Update Credentials from Multiple rotation_items + +**Pre-requisites:** + +1. `env_inventory_generation_job` must be launched in the pipeline run. +2. Environment Instance contains all target Namespace and Application objects referenced by the payload. +3. The payload contains multiple `rotation_items`. +4. The payload includes items from different supported contexts: + + - `pipeline` + - `deployment` + - `runtime` + +5. Each payload item references an existing sensitive parameter linked via the `cred` macro. +6. For each payload item, the target credential is not linked to other parameters. +7. Any payload item with `context: pipeline` does not specify `application`, because Application-level pipeline rotation is rejected by the current implementation. + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: /` +2. `CRED_ROTATION_PAYLOAD: ` +3. `CRED_ROTATION_FORCE: false` + +**Steps:** + +1. The `credential_rotation` job runs in the pipeline: + 1. Reads all `rotation_items` from `CRED_ROTATION_PAYLOAD`. + 2. Processes payload items one by one in the order they are provided. + 3. For each item, chooses the parameter section according to the requested context: + 1. `pipeline` uses `e2eParameters` + 2. `deployment` uses `deployParameters` + 3. `runtime` uses `technicalConfigurationParameters` + 4. For each payload item, searches for affected credentials linked to the same `cred-id` and credential field. + 5. Finds no affected credentials for the successful path and continues processing. + 6. Updates credential values for all valid payload items. + 7. Stops the whole job if any payload item is invalid or cannot be processed. + +**Results:** + +1. All target credential values are updated successfully. +2. The job completes with success status. + +## Affected Credential Handling + +This section covers scenarios where the target parameter shares the same credential reference with other sensitive parameters in the same or different environments. This is the main currently implemented execution path. + +### Affected Credentials with Non-Force Mode + +This group covers scenarios where dependencies are found and `CRED_ROTATION_FORCE=false`. In these cases, the job fails and generates affected parameters artifact. + +### UC-CR-LCH-1: Reject Affected Credential Update + +**Pre-requisites:** + +1. `env_inventory_generation_job` must be launched in the pipeline run. +2. The target sensitive parameter exists and is linked via the `cred` macro. +3. One or more additional sensitive parameters reference the same `cred-id` and the same credential field (`username`, `password`, or `secret`). +4. The linked parameters may be located in: + + - The same Environment Credentials file + - One or more Shared Credentials files + - Other affected Environment Instances + +5. `CRED_ROTATION_PAYLOAD` contains a valid rotation request. +6. `CRED_ROTATION_FORCE` is not provided or is explicitly set to `false`. + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: /` +2. `CRED_ROTATION_PAYLOAD: ` +3. `CRED_ROTATION_FORCE: false` + +**Steps:** + +1. The `credential_rotation` job runs in the pipeline: + 1. Resolves the target parameter and the credential field linked to it. + 2. Finds all other parameters affected by the same credential change. + 3. Builds the full affected parameters report for the request. + 4. Checks `CRED_ROTATION_FORCE` and sees that force mode is disabled. + 5. Generates `affected-sensitive-parameters.yaml` artifact and finishes the job with error status without writing credential changes. + +**Results:** + +1. The `credential_rotation` job fails with a readable error message explaining that affected parameters exist. +2. No credential values are changed. +3. Repository state remains unchanged for all credential files involved in the request. +4. The `affected-sensitive-parameters.yaml` artifact is generated and lists all detected affected parameters. + +### UC-CR-LCH-2: Update Affected Credentials in Force Mode + +**Pre-requisites:** + +1. `env_inventory_generation_job` must be launched in the pipeline run. +2. The target sensitive parameter exists and is linked via the `cred` macro. +3. One or more additional sensitive parameters reference the same `cred-id` and the same credential field. +4. The linked parameters span at least one of the following storage locations: + + - Environment Credentials file of the target Environment + - Shared Credentials file + - Environment Credentials file of another affected Environment + +5. `CRED_ROTATION_PAYLOAD` contains a valid rotation request. +6. `CRED_ROTATION_FORCE` is set to `true`. + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: /` +2. `CRED_ROTATION_PAYLOAD: ` +3. `CRED_ROTATION_FORCE: true` + +**Steps:** + +1. The `credential_rotation` job runs in the pipeline: + 1. Resolves the target parameter and the credential field linked to it. + 2. Finds all other parameters affected by the same credential change. + 3. Builds the full affected parameters report for the request. + 4. Checks `CRED_ROTATION_FORCE` and allows the rotation to continue. + 5. Updates all matched credential files that contain the affected credential. + +**Results:** + +1. The target credential field is updated to the new value. +2. All linked sensitive parameters now reference the rotated value through the shared credential linkage. +3. The `credential_rotation` job completes successfully. +4. The `affected-sensitive-parameters.yaml` artifact is generated. + +### UC-CR-VAL-1: Fail When No Affected Parameters Found + +**Pre-requisites:** + +1. `env_inventory_generation_job` must be launched in the pipeline run. +2. `CRED_ROTATION_PAYLOAD` contains one or more valid `rotation_items`. +3. Each payload item points to an existing sensitive parameter linked via the `cred` macro. +4. None of the payload items has other affected parameters in the current implementation search scope. +5. `CRED_ROTATION_FORCE` is `true`. + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: /` +2. `CRED_ROTATION_PAYLOAD: ` +3. `CRED_ROTATION_FORCE: true` + +**Steps:** + +1. The `credential_rotation` job runs in the pipeline: + 1. Processes all payload items from `CRED_ROTATION_PAYLOAD`. + 2. Tries to collect affected parameters for each payload item. + 3. Finishes payload processing without collecting any affected parameters. + 4. The job finishes with error status. + +**Results:** + +1. The `credential_rotation` job fails. +2. No credential files are updated. +3. `affected-sensitive-parameters.yaml` is not created. + +## Encryption Processing + +This section covers credential rotation behavior that is confirmed by the current implementation. + +### Successful Update with Encryption Enabled + +This group covers successful scenarios when encryption is enabled in `config.yml`. + +### UC-CR-ENC-1: Update Credentials with Plaintext Payload when Encryption Is Enabled + +**Pre-requisites:** + +1. `env_inventory_generation_job` must be launched in the pipeline run. +2. The configuration file `/configuration/config.yml` contains `crypt: true`. +3. The target sensitive parameter exists and is linked via the `cred` macro. +4. `CRED_ROTATION_PAYLOAD` contains plaintext JSON in string form. +5. `CRED_ROTATION_FORCE=true`. +6. One or more additional sensitive parameters reference the same `cred-id` and the same credential field. + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: /` +2. `CRED_ROTATION_PAYLOAD: ` +3. `CRED_ROTATION_FORCE: true` + +**Steps:** + +1. The `credential_rotation` job runs in the pipeline: + 1. Reads the payload in plaintext form. + 2. Loads credential files for the Environment and decrypts them. + 3. Searches for affected credentials linked to the same `cred-id` and credential field. + 4. Finds no affected credentials for the successful path and continues processing. + 5. Applies the requested credential changes to the matched files. + 6. Re-encrypts updated credential files before finishing the job. + +**Results:** + +1. The plaintext payload is processed successfully. +2. Credential values are updated according to payload input. +3. Updated credential files remain encrypted in the repository. +4. The `credential_rotation` job completes with success status. + +### UC-CR-ENC-2: Update Credentials with Encrypted Payload when Encryption Is Enabled + +**Pre-requisites:** + +1. `env_inventory_generation_job` must be launched in the pipeline run. +2. The configuration file `/configuration/config.yml` contains `crypt: true`, so encryption mode is enabled. +3. The target sensitive parameter exists and is linked via the `cred` macro. +4. `CRED_ROTATION_PAYLOAD` is passed in encrypted form and can be decrypted by EnvGene. +5. `CRED_ROTATION_FORCE=true`. +6. One or more additional sensitive parameters reference the same `cred-id` and the same credential field. + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: /` +2. `CRED_ROTATION_PAYLOAD: ` +3. `CRED_ROTATION_FORCE: true` + +**Steps:** + +1. The `credential_rotation` job runs in the pipeline: + 1. Decrypts the payload for processing. + 2. Loads credential files for the Environment and decrypts them. + 3. Searches for affected credentials linked to the same `cred-id` and credential field. + 4. Finds no affected credentials for the successful path and continues processing. + 5. Applies the requested credential changes to the matched files. + 6. Re-encrypts updated credential files before finishing the job. + +**Results:** + +1. The encrypted payload is processed successfully. +2. Credential values are updated according to payload input. +3. Updated credential files remain encrypted in the repository. +4. The `credential_rotation` job completes with success status. + +### Successful Update with Encryption Disabled + +This group covers successful scenarios when encryption is disabled in `config.yml`. + +### UC-CR-ENC-3: Update Credentials with Plaintext Payload when Encryption Is Disabled + +**Pre-requisites:** + +1. `env_inventory_generation_job` must be launched in the pipeline run. +2. The configuration file `/configuration/config.yml` contains `crypt: false`. +3. The target sensitive parameter exists and is linked via the `cred` macro. +4. `CRED_ROTATION_PAYLOAD` contains plaintext JSON in string form. +5. `CRED_ROTATION_FORCE=true`. +6. One or more additional sensitive parameters reference the same `cred-id` and the same credential field. + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: /` +2. `CRED_ROTATION_PAYLOAD: ` +3. `CRED_ROTATION_FORCE: true` + +**Steps:** + +1. The `credential_rotation` job runs in the pipeline: + 1. Reads the payload in plaintext form. + 2. Loads credential files for the Environment without repository decryption. + 3. Searches for affected credentials linked to the same `cred-id` and credential field. + 4. Finds no affected credentials for the successful path and continues processing. + 5. Applies the requested credential changes to the matched files. + 6. Finishes without credential file re-encryption. + +**Results:** + +1. The plaintext payload is processed successfully. +2. Credential values are updated according to payload input. +3. The `credential_rotation` job completes with success status. + +### UC-CR-ENC-4: Update Credentials with Encrypted Payload when Encryption Is Disabled + +**Pre-requisites:** + +1. `env_inventory_generation_job` must be launched in the pipeline run. +2. The EnvGene configuration file `/configuration/config.yml` contains `crypt: false`, so encryption mode is disabled. +3. The target sensitive parameter exists and is linked via the `cred` macro. +4. `CRED_ROTATION_PAYLOAD` is passed in encrypted form and can be decrypted by EnvGene. + +5. `CRED_ROTATION_FORCE=true`. +6. One or more additional sensitive parameters reference the same `cred-id` and the same credential field. +7. The rotation request is otherwise valid. + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: /` +2. `CRED_ROTATION_PAYLOAD: ` + +**Steps:** + +1. The `credential_rotation` job runs in the pipeline: + 1. Reads the payload and decrypts it for further processing. + 2. Loads credential files for the Environment without repository decryption. + 3. Searches for affected credentials linked to the same `cred-id` and credential field. + 4. Finds no affected credentials for the successful path and continues processing. + 5. Applies the requested credential changes to the matched files. + 6. Finishes without credential file re-encryption. + +**Results:** + +1. The encrypted payload is processed successfully. +2. Credential values are updated according to payload input. +3. The `credential_rotation` job completes with success status. diff --git a/docs/use-cases/env-template-downloading.md b/docs/use-cases/env-template-downloading.md deleted file mode 100644 index c7d08cac5..000000000 --- a/docs/use-cases/env-template-downloading.md +++ /dev/null @@ -1,69 +0,0 @@ -# Environment Template Downloading Use Cases - -- [Environment Template Downloading Use Cases](#environment-template-downloading-use-cases) - - [Template Artifact Downloading Details](#template-artifact-downloading-details) - - [Use Cases](#use-cases) - -## Template Artifact Downloading Details - -The process for downloading environment template artifacts in EnvGene can be categorized along four primary axes: - -1. **Version Type:** Explicit version or `SNAPSHOT` version (latest available) - -2. **Artifact Coordinate Notation:** Either GAV (Group, Artifact, Version) or app:ver notation. - - Template artifact can be specified in [Environment Inventory](/docs/envgene-configs.md#env_definitionyml) with: - - 1. app:ver notation. To use this, [Artifact Definition](/docs/envgene-objects.md#artifact-definition) is needed. - - Example of `env_definition.yml`: - - ```yaml - - envTemplate: - artifact: string - ``` - - 2. GAV notation. To use this, [registry.yaml](/docs/envgene-configs.md#registryyml) is needed. - - Example of `env_definition.yml`: - - ```yaml - templateArtifact: - registry: string - repository: string - templateRepository: string - artifact: - group_id: string - artifact_id: string - version: string - ``` - -3. **Artifact Source Registry:** Artifact repositories supported include: - - 1. Artifactory - 2. Nexus - 3. AWS CodeArtifact - 4. Azure Artifacts - 5. GCP Artifact Registry - - The GAV form is limited to Artifactory/Nexus, while app:ver supports all. - -4. **Artifact Content Type:** Either a ZIP archive or Delivery Unit (DU) - - > [!NOTE] The core EnvGene does not have built-in support for Delivery Unit (DU) processing; this is provided as an extension point. - -## Use Cases - -The use cases below enumerate combinations of these axes that EnvGene supports: - -| Use Case | Coordinate Notation | Version Type | Registry Scope | Artifact Type | Prerequisites | Result | -|:--------:|:-------------------:|:------------:|:----------------------:|:-------------:|:-----------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------| -| UC-1-1 | GAV | Specific | Artifactory/Nexus | ZIP | `registry.yml` must include registry configuration | Retrieve a ZIP artifact by explicit GAV coordinates and fixed version from Artifactory or Nexus. | -| UC-1-2 | GAV | Specific | Artifactory/Nexus | DU | `registry.yml` must include registry configuration | Retrieve a DU artifact by explicit GAV coordinates and fixed version from Artifactory or Nexus. | -| UC-1-3 | GAV | SNAPSHOT | Artifactory/Nexus | ZIP | `registry.yml` must include registry configuration | Retrieve the latest available ZIP artifact by GAV coordinates with SNAPSHOT version from Artifactory or Nexus. | -| UC-1-4 | GAV | SNAPSHOT | Artifactory/Nexus | DU | `registry.yml` must include registry configuration | Retrieve the latest available DU artifact by GAV coordinates with SNAPSHOT version from Artifactory or Nexus. | -| UC-2-1 | app:ver | Specific | Any supported registry | ZIP | Artifact Definition must exist for the specified app:ver | Retrieve a ZIP artifact by explicit app:ver notation and fixed version from any supported registry. | -| UC-2-2 | app:ver | Specific | Any supported registry | DU | Artifact Definition must exist for the specified app:ver | Retrieve a DU artifact by explicit app:ver notation and fixed version from any supported registry. | -| UC-2-3 | app:ver | SNAPSHOT | Any supported registry | ZIP | Artifact Definition must exist for the specified app:ver | Retrieve the latest available ZIP artifact by app:ver notation with SNAPSHOT version from any registry. | -| UC-2-4 | app:ver | SNAPSHOT | Any supported registry | DU | Artifact Definition must exist for the specified app:ver | Retrieve the latest available DU artifact by app:ver notation with SNAPSHOT version from any registry. | diff --git a/docs/use-cases/environment-instance-generation.md b/docs/use-cases/environment-instance-generation.md index 43c91f22d..506adae3e 100644 --- a/docs/use-cases/environment-instance-generation.md +++ b/docs/use-cases/environment-instance-generation.md @@ -15,6 +15,13 @@ - [UC-EIG-TA-1: Environment Instance Generation with `artifact` only](#uc-eig-ta-1-environment-instance-generation-with-artifact-only) - [UC-EIG-TA-2: Environment Instance Generation with `artifact` and `bgNsArtifacts` and BG Domain](#uc-eig-ta-2-environment-instance-generation-with-artifact-and-bgnsartifacts-and-bg-domain) - [UC-EIG-TA-3: Environment Instance Generation with `artifact` and `bgNsArtifacts` and without BG Domain](#uc-eig-ta-3-environment-instance-generation-with-artifact-and-bgnsartifacts-and-without-bg-domain) + - [Effective Set Generation in Instance Pipeline](#effective-set-generation-in-instance-pipeline) + - [UC-EIG-ES-1: Generate Effective Set without `SD_DATA` or `SD_VERSION`](#uc-eig-es-1-generate-effective-set-without-sd_data-or-sd_version) + - [UC-EIG-ES-2: Generate Effective Set with `SD_DATA` or `SD_VERSION`](#uc-eig-es-2-generate-effective-set-with-sd_data-or-sd_version) + - [UC-EIG-ES-3: Apply `CUSTOM_PARAMS` when `GENERATE_EFFECTIVE_SET` is true](#uc-eig-es-3-apply-custom_params-when-generate_effective_set-is-true) + - [UC-EIG-ES-4: Ignore `CUSTOM_PARAMS` when `GENERATE_EFFECTIVE_SET` is false](#uc-eig-es-4-ignore-custom_params-when-generate_effective_set-is-false) + - [Multiple Environments Processing](#multiple-environments-processing) + - [UC-EIG-ME-1: Parallel Environment Instance Generation for Multiple Environments](#uc-eig-me-1-parallel-environment-instance-generation-for-multiple-environments) ## Overview @@ -447,3 +454,181 @@ Instance pipeline (GitLab or GitHub) is started with parameters: 1. All Namespaces are rendered using `project-env-template:v1.2.3` 2. All other objects (Tenant, Cloud, Applications, etc.) are rendered using `project-env-template:v1.2.3` 3. `bgNsArtifacts` are ignored since BG Domain is absent + +## Effective Set Generation in Instance Pipeline + +This section describes Effective Set generation scenarios executed from the Instance pipeline. + +### UC-EIG-ES-1: Generate Effective Set without `SD_DATA` or `SD_VERSION` + +**Pre-requisites:** + +1. Environment Inventory exists and can be processed by the Instance pipeline. +2. Pipeline parameters include: + 1. `ENV_BUILDER: true` + 2. `GENERATE_EFFECTIVE_SET: true` + 3. `SD_DATA` is empty or not set + 4. `SD_VERSION` is empty or not set + +**Trigger:** + +Instance build pipeline with Effective Set enabled is started for one or more environments. + +**Steps:** + +1. The `env_builder` job runs and generates Environment Instance objects. +2. The `process_sd` job is skipped because neither `SD_DATA` nor `SD_VERSION` is provided. +3. The `generate_effective_set` job runs without SD input parameters. + +**Results:** + +1. `env_builder` job finishes successfully. +2. `generate_effective_set` job finishes successfully. +3. Generated Effective Set does not include data merged from SD input. + +### UC-EIG-ES-2: Generate Effective Set with `SD_DATA` or `SD_VERSION` + +**Pre-requisites:** + +1. Environment Inventory exists and can be processed by the Instance pipeline. +2. Pipeline parameters include: + 1. `ENV_BUILDER: true` + 2. `GENERATE_EFFECTIVE_SET: true` + 3. At least one SD input is set: + - `SD_DATA`, or + - `SD_VERSION` + +**Trigger:** + +Instance build pipeline with Effective Set enabled is started for one or more environments. + +**Steps:** + +1. The `env_builder` job runs and generates Environment Instance objects. +2. The `process_sd` job runs when SD input matches `SD_SOURCE_TYPE`: + 1. `SD_SOURCE_TYPE=json` and `SD_DATA` is provided, or + 2. `SD_SOURCE_TYPE=artifact` and `SD_VERSION` is provided. +3. The `generate_effective_set` job runs with processed SD input. + +**Results:** + +1. `env_builder` job finishes successfully. +2. `process_sd` job runs and finishes successfully. +3. `generate_effective_set` job finishes successfully. +4. Generated Effective Set includes data resolved from provided SD input. + +### UC-EIG-ES-3: Apply `CUSTOM_PARAMS` when `GENERATE_EFFECTIVE_SET` is true + +**Pre-requisites:** + +1. Environment Inventory exists and can be processed by the Instance pipeline. +2. Pipeline parameters include: + 1. `ENV_BUILDER: true` + 2. `GENERATE_EFFECTIVE_SET: true` + 3. `CUSTOM_PARAMS` contains one or more key-value pairs + +**Trigger:** + +Instance build pipeline with Effective Set enabled is started for one or more environments. + +**Steps:** + +1. The `env_builder` job runs and generates Environment Instance objects. +2. The `generate_effective_set` job runs. +3. The `generate_effective_set` job applies values from `CUSTOM_PARAMS`. + +**Results:** + +1. `env_builder` job finishes successfully. +2. `generate_effective_set` job finishes successfully. +3. Values from `CUSTOM_PARAMS` are applied in generated Effective Set according to merge rules. + +### UC-EIG-ES-4: Ignore `CUSTOM_PARAMS` when `GENERATE_EFFECTIVE_SET` is false + +**Pre-requisites:** + +1. Environment Inventory exists and can be processed by the Instance pipeline. +2. Pipeline parameters include: + 1. `ENV_BUILDER: true` + 2. `GENERATE_EFFECTIVE_SET: false` + 3. `CUSTOM_PARAMS` contains one or more key-value pairs + +**Trigger:** + +Instance build pipeline is started for one or more environments with: + +1. `GENERATE_EFFECTIVE_SET: false` + +**Steps:** + +1. The `env_builder` job runs and generates Environment Instance objects. +2. The `generate_effective_set` job is skipped because `GENERATE_EFFECTIVE_SET` is false. + +**Results:** + +1. `env_builder` job finishes successfully. +2. `generate_effective_set` job is not created. +3. Environment Instance generation completes without Effective Set output. +4. `CUSTOM_PARAMS` are ignored. + +## Multiple Environments Processing + +When multiple environments are specified in `ENV_NAMES`, the pipeline processes them in parallel. Each environment triggers an independent pipeline flow with the same set of pipeline parameters. This section describes how Environment Instance generation works for multiple environments. + +### UC-EIG-ME-1: Parallel Environment Instance Generation for Multiple Environments + +**Pre-requisites:** + +1. Multiple Environment Inventories exist +2. Each Environment Inventory contains valid Environment Template configuration +3. Template artifacts are available for all environments + +**Trigger:** + +> [!Note] +> One of the following conditions must be met. + +1. GitLab Instance pipeline is started with parameters: + 1. `ENV_NAMES: \n` + 2. `ENV_BUILDER: true` + + Or using semicolon separator: + 1. `ENV_NAMES: ;` + 2. `ENV_BUILDER: true` + + Or using comma separator: + 1. `ENV_NAMES: ,` + 2. `ENV_BUILDER: true` + + Or using space separator: + 1. `ENV_NAMES: ` + 2. `ENV_BUILDER: true` +2. GitHub Instance pipeline is started with parameters: + 1. `ENV_NAMES: \n` + 2. `ENV_BUILDER: true` + + Or using semicolon separator: + 1. `ENV_NAMES: ;` + 2. `ENV_BUILDER: true` + + Or using comma separator: + 1. `ENV_NAMES: ,` + 2. `ENV_BUILDER: true` + + Or using space separator: + 1. `ENV_NAMES: ` + 2. `ENV_BUILDER: true` + +**Steps:** + +1. For each environment from the list, parallel and independent pipeline flows are started +2. For each environment, the `env_builder` job runs in parallel: + 1. Reads Environment Inventory for the specific environment + 2. Generates Environment Instance objects according to the template configuration + +**Results:** + +1. Environment Instance for each environment from `ENV_NAMES` is generated and saved to `/environments///` +2. All Environment Instances are generated in parallel, independently of each other +3. Each Environment Instance generation uses the same pipeline parameters but processes its own Environment Inventory +4. If one environment generation fails, other environments continue processing independently diff --git a/docs/use-cases/environment-inventory-generation.md b/docs/use-cases/environment-inventory-generation.md index e3ac0ba57..37162b22b 100644 --- a/docs/use-cases/environment-inventory-generation.md +++ b/docs/use-cases/environment-inventory-generation.md @@ -336,6 +336,7 @@ Instance pipeline (GitLab or GitHub) is started with: - `action: create_or_replace` - `place: env | cluster | site` +- `name: ` - `content` is a credentials map (one or multiple credentials) **Steps:** @@ -346,6 +347,7 @@ Instance pipeline (GitLab or GitHub) is started with: - [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) - `action == create_or_replace` - `place ∈ { env, cluster, site }` + - `name` is present - `content` is present 3. Resolves target path by `place`: - `place=env` → `/environments///Inventory/credentials/inventory_generation_creds.yml` @@ -382,6 +384,7 @@ Instance pipeline (GitLab or GitHub) is started with: - `action: create_or_replace` - `place: env | cluster | site` +- `name: ` - `content` is a credentials map (one or multiple credentials) **Steps:** @@ -392,6 +395,7 @@ Instance pipeline (GitLab or GitHub) is started with: - [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) - `action == create_or_replace` - `place ∈ { env, cluster, site }` + - `name` is present - `content` is present 3. Resolves target path by `place`. 4. Replaces the credentials file using `content` (fully overwrites the file). @@ -424,6 +428,7 @@ Instance pipeline (GitLab or GitHub) is started with: - `action: delete` - `place: env | cluster | site` +- `name: ` - `content` is present **Steps:** @@ -433,6 +438,7 @@ Instance pipeline (GitLab or GitHub) is started with: 2. Validates the `credentials[]` item against the request schema: - `action == delete` - `place ∈ { env, cluster, site }` + - `name` is present - `content` is present 3. Resolves target credentials file path by `place`. 4. Deletes the target credentials file if it exists. diff --git a/docs/use-cases/gsf-repository-maintenance.md b/docs/use-cases/gsf-repository-maintenance.md new file mode 100644 index 000000000..763356218 --- /dev/null +++ b/docs/use-cases/gsf-repository-maintenance.md @@ -0,0 +1,258 @@ +# GSF Repository Maintenance Use Cases + +- [GSF Repository Maintenance Use Cases](#gsf-repository-maintenance-use-cases) + - [Overview](#overview) + - [Template Repository Maintenance via GSF](#template-repository-maintenance-via-gsf) + - [UC-GSF-TMP-1: Initialize Template Repository via GSF](#uc-gsf-tmp-1-initialize-template-repository-via-gsf) + - [UC-GSF-TMP-2: Upgrade Template Repository via GSF](#uc-gsf-tmp-2-upgrade-template-repository-via-gsf) + - [UC-GSF-TMP-3: Downgrade Template Repository via GSF](#uc-gsf-tmp-3-downgrade-template-repository-via-gsf) + - [Instance Repository Maintenance via GSF](#instance-repository-maintenance-via-gsf) + - [UC-GSF-INST-1: Initialize Instance Repository via GSF](#uc-gsf-inst-1-initialize-instance-repository-via-gsf) + - [UC-GSF-INST-2: Upgrade Instance Repository via GSF](#uc-gsf-inst-2-upgrade-instance-repository-via-gsf) + - [UC-GSF-INST-3: Downgrade Instance Repository via GSF](#uc-gsf-inst-3-downgrade-instance-repository-via-gsf) + +## Overview + +This document defines use cases for maintaining EnvGene Template and Instance repositories with the Git-System-Follower (GSF) package manager. + +For each repository type, it covers three maintenance scenarios: + +- Initial installation (init) +- Upgrade to a new EnvGene package version +- Downgrade to an older EnvGene package version + +For every scenario, repository contents after GSF execution are validated against a reference structure (golden state) to confirm that managed files are correctly added, updated, or removed. + +For detailed installation and maintenance steps, see: + +- Template Repository: [Maintain Template Repository via GSF] +- Instance Repository: [Environment Instance Repository Installation Guide](/docs/how-to/envgene-maitanance.md) + +## Template Repository Maintenance via GSF + +### UC-GSF-TMP-1: Initialize Template Repository via GSF + +**Pre-requisites:** + +1. A new Git repository for the Environment Template exists in the project Git group and does not yet contain EnvGene-specific files. +2. GitLab technical user and access token with required permissions are available. +3. GSF package manager is installed and working on the local machine. +4. Template package image path for the desired EnvGene version is known. +5. A reference (target) Template Repository structure for this version is defined. + +**Trigger:** + +User runs GSF on the local machine to initialize the Template Repository: + +```bash +git-system-follower install \ + -r \ + -b \ + -t \ + --extra env_template_artifact_name no-masked +``` + +**Steps:** + +1. Run GSF with repository URL, branch, token, and package image. +2. GSF applies the selected package to the Template Repository. +3. GSF adds required files from the selected version and removes obsolete managed files (if any). + +**Results:** + +1. Template Repository is initialized. +2. Required files from the selected version are present. +3. Old managed files are removed or replaced. +4. Repository matches the reference structure. + +### UC-GSF-TMP-2: Upgrade Template Repository via GSF + +**Pre-requisites:** + +1. Template Repository already exists and contains a previous EnvGene template package version. +2. GitLab technical user, token, and required CI/CD variables are available. +3. GSF package manager is installed and working on the local machine. +4. Target EnvGene template package image path is known. +5. A reference Template Repository structure for the target EnvGene version is defined. + +**Trigger:** + +User runs GSF on the local machine to upgrade the Template Repository to a new EnvGene version: + +```bash +git-system-follower install \ + -r \ + -b \ + -t \ + --extra env_template_artifact_name no-masked +``` + +**Steps:** + +1. Run GSF with repository URL, branch, token, and target package image. +2. GSF updates the Template Repository to the target version. +3. GSF updates changed files, adds new files, and removes outdated managed files. +4. Verify restricted files for Template Repository: + - `pipeline_vars.yml` or `pipeline_vars.yaml` + - `build_vars.sh` + - `description_template.yml` or `description_template.yaml` +5. Verify restricted file behavior: + - `build_vars.sh` and `description_template.*` preserve user-defined values after upgrade + - `pipeline_vars.*` preserves user-defined values, except allowed structural alignment with current package structure + +**Results:** + +1. Template Repository is upgraded to the target version. +2. Repository matches the reference structure. +3. Restricted files are preserved according to policy: + - `build_vars.sh` and `description_template.*` are not replaced with package defaults + - `pipeline_vars.*` is preserved, with structural alignment allowed when required +4. No regressions related to repository upgrade are observed. + +### UC-GSF-TMP-3: Downgrade Template Repository via GSF + +**Pre-requisites:** + +1. Template Repository already exists and has a later version. +2. GitLab token and required variables are available. +3. GSF is installed on the local machine. +4. Path to an older template package image is known. +5. Reference structure for the older version is available. + +**Trigger:** + +User runs GSF on the local machine to install an older template package version: + +```bash +git-system-follower install \ + -r \ + -b \ + -t \ + --extra env_template_artifact_name no-masked +``` + +**Steps:** + +1. Run GSF with repository URL, branch, token, and older package image. +2. GSF applies the older package to the Template Repository. +3. GSF replaces current managed files with older-version files and removes extra files. + +**Results:** + +1. Template Repository is switched to the older version. +2. Required files for the older version are present. +3. Files from the later version are removed. +4. Repository matches the reference structure. + +## Instance Repository Maintenance via GSF + +### UC-GSF-INST-1: Initialize Instance Repository via GSF + +**Pre-requisites:** + +1. A new Git repository for the Environment Instance exists in the project Git group. +2. GitLab project access token with required scopes is available. +3. GSF package manager is installed and working on the local machine. +4. Instance package image path for the chosen EnvGene version is known. +5. A reference Instance Repository structure for this version is defined. + +**Trigger:** + +User runs GSF on the local machine to initialize the Instance Repository: + +```bash +git-system-follower install \ + -r \ + -b \ + -t +``` + +**Steps:** + +1. Run GSF with repository URL, branch, token, and package image. +2. GSF applies the selected package to the Instance Repository. +3. GSF creates the required CI/CD and configuration files for the selected version. + +**Results:** + +1. Instance Repository is initialized. +2. Required files from the selected version are present. +3. Repository matches the reference structure. + +### UC-GSF-INST-2: Upgrade Instance Repository via GSF + +**Pre-requisites:** + +1. Instance Repository already exists and contains a previous EnvGene instance package version. +2. GitLab token and required CI/CD variables are available. +3. GSF package manager is installed and working on the operator's machine. +4. Target EnvGene instance package image path is known. +5. A reference Instance Repository structure for the target EnvGene version is defined. + +**Trigger:** + +User runs GSF on the local machine to upgrade the Instance Repository: + +```bash +git-system-follower install \ + -r \ + -b \ + -t +``` + +**Steps:** + +1. Run GSF with repository URL, branch, token, and target package image. +2. GSF updates the Instance Repository to the target version. +3. GSF updates changed files, adds new files, and removes outdated managed files. +4. Verify `configuration/credentials/credentials.yml` contains `self-token-cred`: + - `type: secret` + - `data.secret` is present. +5. Verify `configuration/integration.yml` contains: + - `self_token: "${creds.get('self-token-cred').secret}"`. +6. Verify legacy `self_token` definition is absent in `configuration/config.yml` or ignored by the target version. +7. Verify placeholder file `configuration/.gitkeep` is present. + +**Results:** + +1. Instance Repository is upgraded to the target version. +2. Repository matches the reference structure. +3. Token configuration is migrated and valid: + - `configuration/credentials/credentials.yml` contains `self-token-cred` with non-empty secret data, + - `configuration/integration.yml` references `${creds.get('self-token-cred').secret}` in `self_token`, + - runtime execution does not fail with missing `self_token` or missing `self-token-cred`. +4. Legacy token definition in `configuration/config.yml` is absent or ignored by the target version, and does not affect runtime behavior. + +### UC-GSF-INST-3: Downgrade Instance Repository via GSF + +**Pre-requisites:** + +1. Instance Repository already exists and has a later version. +2. GitLab token and required variables are available. +3. GSF is installed on the local machine. +4. Path to an older instance package image is known. +5. Reference structure for the older version is available. + +**Trigger:** + +User runs GSF on the local machine to install an older instance package version: + +```bash +git-system-follower install \ + -r \ + -b \ + -t +``` + +**Steps:** + +1. Run GSF with repository URL, branch, token, and older package image. +2. GSF applies the older package to the Instance Repository. +3. GSF replaces current managed files with older-version files and removes extra files. + +**Results:** + +1. Instance Repository is switched to the older version. +2. Required files for the older version are present. +3. Files from the later version are removed. +4. Repository matches the reference structure. diff --git a/docs/use-cases/sbom-retention.md b/docs/use-cases/sbom-retention.md new file mode 100644 index 000000000..ed93f0e08 --- /dev/null +++ b/docs/use-cases/sbom-retention.md @@ -0,0 +1,204 @@ +# SBOM Retention Use Cases + +- [SBOM Retention Use Cases](#sbom-retention-use-cases) + - [Overview](#overview) + - [SBOM Cleanup Execution](#sbom-cleanup-execution) + - [UC-SBOM-1: SBOM Retention Disabled - No Cleanup](#uc-sbom-1-sbom-retention-disabled---no-cleanup) + - [UC-SBOM-2: Repository Below Threshold - No Cleanup](#uc-sbom-2-repository-below-threshold---no-cleanup) + - [UC-SBOM-3: Repository Above Threshold - Cleanup with Default Settings](#uc-sbom-3-repository-above-threshold---cleanup-with-default-settings) + - [UC-SBOM-4: Repository Above Threshold - Cleanup with Custom Version Count](#uc-sbom-4-repository-above-threshold---cleanup-with-custom-version-count) + +## Overview + +This document covers use cases for [SBOM Retention](/docs/features/sbom-retention.md) - automatic cleanup of cached SBOM files to manage Instance Repository size. SBOM files are stored under `/sboms//` as `-.sbom.json` (see [SBOM directory layout](/docs/features/sbom.md#sbom-directory-layout)). + +## SBOM Cleanup Execution + +The cleanup logic is triggered during effective set generation and depends on configuration settings and repository size. These use cases demonstrate different scenarios based on configuration and repository state. + +### UC-SBOM-1: SBOM Retention Disabled - No Cleanup + +**Pre-requisites:** + +1. Instance Repository exists with `/sboms/` directory; SBOM files are stored in `/sboms//` +2. SBOM files exist in `/sboms//` +3. SBOM retention is **disabled** in `/configuration/config.yml`: + + ```yaml + # Option 1: No sbom_retention section + crypt: false + ``` + + or + + ```yaml + # Option 2: Explicitly disabled + sbom_retention: + enabled: false + ``` + +4. Repository size is 1300 MB (above 1200 MB threshold) + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `GENERATE_EFFECTIVE_SET: true` + +**Steps:** + +1. The `generate_effective_set` job runs in the pipeline: + 1. Generates effective set for the environment + 2. Checks SBOM retention configuration + 3. Finds `sbom_retention.enabled: false` (or no configuration) + 4. Skips SBOM cleanup logic + 5. Completes effective set generation + +**Results:** + +1. Effective set is generated successfully +2. No SBOM files are deleted +3. Pipeline log shows: "SBOM retention is disabled, skipping cleanup" + +### UC-SBOM-2: Repository Below Threshold - No Cleanup + +**Pre-requisites:** + +1. Instance Repository exists with `/sboms/` directory; SBOM files are stored in `/sboms//` +2. SBOM files exist in `/sboms//` with total size 800 MB +3. SBOM retention is **enabled** in `/configuration/config.yml`: + + ```yaml + sbom_retention: + enabled: true + keep_versions_per_app: 10 + ``` + +4. Repository size is 800 MB (below 1200 MB threshold) + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `GENERATE_EFFECTIVE_SET: true` + +**Steps:** + +1. The `generate_effective_set` job runs in the pipeline: + 1. Generates effective set for the environment + 2. Checks SBOM retention configuration + 3. Finds `sbom_retention.enabled: true` + 4. Checks repository size: 800 MB + 5. Compares with threshold: 800 MB < 1200 MB + 6. Skips SBOM cleanup (threshold not reached) + 7. Completes effective set generation + +**Results:** + +1. Effective set is generated successfully +2. No SBOM files are deleted +3. Pipeline log shows: "Repository size (800 MB) below threshold (1200 MB), skipping cleanup" + +### UC-SBOM-3: Repository Above Threshold - Cleanup with Default Settings + +**Pre-requisites:** + +1. Instance Repository exists with `/sboms/` directory +2. SBOM files exist for multiple applications under per-application subdirectories: + - `/sboms/app-a/`: `app-a-1.0.15.sbom.json` through `app-a-1.0.1.sbom.json` (15 versions) + - `/sboms/app-b/`: `app-b-2.0.12.sbom.json` through `app-b-2.0.1.sbom.json` (12 versions) + - `/sboms/app-c/`: `app-c-3.5.8.sbom.json` through `app-c-3.5.1.sbom.json` (8 versions) +3. SBOM retention is **enabled** with default settings in `/configuration/config.yml`: + + ```yaml + sbom_retention: + enabled: true + keep_versions_per_app: 10 # default value + ``` + +4. Repository size is 1300 MB (above 1200 MB threshold) + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `GENERATE_EFFECTIVE_SET: true` + +**Steps:** + +1. The `generate_effective_set` job runs in the pipeline: + 1. Generates effective set for the environment + 2. Checks SBOM retention configuration: enabled with `keep_versions_per_app: 10` + 3. Checks repository size: 1300 MB > 1200 MB threshold + 4. Triggers SBOM cleanup process: + 1. Scans `/sboms/` directory (each subdirectory is one application) + 2. For each application subdirectory (e.g. `/sboms/app-a/`): + - Sorts versions by file creation time (newest first) + - Keeps 10 most recent versions + - Deletes older versions + 5. Completes effective set generation + +**Results:** + +1. Effective set is generated successfully +2. SBOM files are cleaned up for each application: + - **app-a** (under `/sboms/app-a/`): Keeps versions 1.0.15 through 1.0.6 (10 versions) + - **app-b** (under `/sboms/app-b/`): Keeps versions 2.0.12 through 2.0.3 (10 versions) + - **app-c** (under `/sboms/app-c/`): Keeps all 8 versions (no deletion needed) +3. Total files deleted: 7 SBOM files +4. Pipeline log shows: + - "Repository size (1300 MB) above threshold (1200 MB), starting cleanup" + - "Cleaned up 7 SBOM files" + - "Kept 10 versions per application" + +### UC-SBOM-4: Repository Above Threshold - Cleanup with Custom Version Count + +**Pre-requisites:** + +1. Instance Repository exists with `/sboms/` directory +2. SBOM files exist for application `postgres` under `/sboms/postgres/`: + - `postgres-pg16-2.10.10.sbom.json` through `postgres-pg16-2.10.1.sbom.json` (10 versions) +3. SBOM retention is **enabled** with custom settings in `/configuration/config.yml`: + + ```yaml + sbom_retention: + enabled: true + keep_versions_per_app: 3 # only keep 3 most recent versions + ``` + +4. Repository size is 1350 MB (above 1200 MB threshold) + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with parameters: + +1. `ENV_NAMES: ` +2. `GENERATE_EFFECTIVE_SET: true` + +**Steps:** + +1. The `generate_effective_set` job runs in the pipeline: + 1. Generates effective set for the environment + 2. Checks SBOM retention configuration: enabled with `keep_versions_per_app: 3` + 3. Checks repository size: 1350 MB > 1200 MB threshold + 4. Triggers SBOM cleanup process: + 1. Scans `/sboms/` directory (finds subdirectory `postgres`) + 2. For `/sboms/postgres/`: sorts versions by file creation time (newest first) + 3. Keeps 3 most recent versions: 2.10.10, 2.10.9, 2.10.8 + 4. Deletes 7 older versions: 2.10.7 through 2.10.1 + 5. Completes effective set generation + +**Results:** + +1. Effective set is generated successfully +2. SBOM files for `postgres` under `/sboms/postgres/` are cleaned up: + - **Kept**: `/sboms/postgres/postgres-pg16-2.10.10.sbom.json`, `postgres-pg16-2.10.9.sbom.json`, `postgres-pg16-2.10.8.sbom.json` + - **Deleted**: `/sboms/postgres/postgres-pg16-2.10.7.sbom.json` through `postgres-pg16-2.10.1.sbom.json` (7 files) +3. Total files deleted: 7 SBOM files +4. Pipeline log shows: + - "Repository size (1350 MB) above threshold (1200 MB), starting cleanup" + - "Cleaned up 7 SBOM files for postgres" + - "Kept 3 versions per application" diff --git a/docs/use-cases/sbom-storage-migration.md b/docs/use-cases/sbom-storage-migration.md new file mode 100644 index 000000000..a3a695672 --- /dev/null +++ b/docs/use-cases/sbom-storage-migration.md @@ -0,0 +1,35 @@ +# SBOM Storage Migration Use Case + +- [SBOM Storage Migration Use Case](#sbom-storage-migration-use-case) + - [Overview](#overview) + - [UC-SBOM-MIG-1: First run after upgrade](#uc-sbom-mig-1-first-run-after-upgrade) + +## Overview + +This document describes the use case for **automatic migration** from the flat SBOM layout to the per-application layout when upgrading EnvGene. For the procedural guide (when to use it, what to expect), see [Migrate SBOM Storage to Per-Application Layout](/docs/how-to/sbom-storage-migration.md). Target layout: [SBOM directory layout](/docs/features/sbom.md#sbom-directory-layout). + +## UC-SBOM-MIG-1: First run after upgrade + +**Pre-requisites:** + +1. Instance Repository has SBOM files in the flat layout: `/sboms/-.sbom.json` +2. EnvGene (Instance pipeline) is upgraded to a version that uses the per-application SBOM layout + +**Trigger:** + +Instance pipeline is run with parameters that trigger effective set generation (e.g. `ENV_NAMES: `, `GENERATE_EFFECTIVE_SET: true`). + +**Steps:** + +1. The pipeline runs with the new EnvGene version. +2. The `generate_effective_set` job (or equivalent) detects the old flat SBOM layout. +3. EnvGene removes all SBOM files from `/sboms/` (flat location). +4. For each application version required by the Solution Descriptor, EnvGene generates the SBOM and writes it to `/sboms//-.sbom.json`. +5. Effective set generation completes using the new SBOM paths. + +**Results:** + +1. No SBOM files remain directly under `/sboms/`; all SBOMs are under `/sboms//`. +2. Effective set is generated successfully. +3. Repository contains the new layout; subsequent runs do not perform migration again. +4. The run may take longer than usual due to SBOM regeneration. diff --git a/docs/use-cases/sd-processing.md b/docs/use-cases/sd-processing.md index 34b5bbe01..832b43013 100644 --- a/docs/use-cases/sd-processing.md +++ b/docs/use-cases/sd-processing.md @@ -329,25 +329,71 @@ The SD processing logic depends on: **Trigger:** > [!Note] -> One of the following conditions must be met: +> One of the following conditions must be met. 1. GitLab Instance pipeline is started with parameters: 1. `ENV_NAMES: ` 2. `SD_VERSION: \n` - Or with explicit parameters: + Or using semicolon separator: + 1. `ENV_NAMES: ` + 2. `SD_VERSION: ;` + + Or using comma separator: + 1. `ENV_NAMES: ` + 2. `SD_VERSION: ,` + + Or using space separator: + 1. `ENV_NAMES: ` + 2. `SD_VERSION: ` + + Or with explicit parameters (using newline): 1. `ENV_NAMES: ` 2. `SD_SOURCE_TYPE: artifact` 3. `SD_VERSION: \n` 4. `SD_REPO_MERGE_MODE: basic-merge` + + Or with explicit parameters (using semicolon): + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ;` + 4. `SD_REPO_MERGE_MODE: basic-merge` + + Or with explicit parameters (using comma): + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ,` + 4. `SD_REPO_MERGE_MODE: basic-merge` + + Or with explicit parameters (using space): + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ` + 4. `SD_REPO_MERGE_MODE: basic-merge` 2. GitHub Instance pipeline is started with parameters: 1. `ENV_NAMES: ` 2. `GH_ADDITIONAL_PARAMS: "SD_VERSION=\\n"` - Or with explicit parameters: + Or using comma separator: + 1. `ENV_NAMES: ` + 2. `GH_ADDITIONAL_PARAMS: "SD_VERSION=,"` + + Or using space separator: + 1. `ENV_NAMES: ` + 2. `GH_ADDITIONAL_PARAMS: "SD_VERSION= "` + + Or with explicit parameters (using newline): 1. `ENV_NAMES: ` 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION=\\n,SD_REPO_MERGE_MODE=basic-merge"` + Or with explicit parameters (using comma): + 1. `ENV_NAMES: ` + 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION=,,SD_REPO_MERGE_MODE=basic-merge"` + + Or with explicit parameters (using space): + 1. `ENV_NAMES: ` + 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION= ,SD_REPO_MERGE_MODE=basic-merge"` + **Steps:** 1. The `process_sd` job runs in the pipeline: @@ -375,25 +421,71 @@ The SD processing logic depends on: **Trigger:** > [!Note] -> One of the following conditions must be met: +> One of the following conditions must be met. 1. GitLab Instance pipeline is started with parameters: 1. `ENV_NAMES: ` 2. `SD_VERSION: \n` - Or with explicit parameters: + Or using semicolon separator: + 1. `ENV_NAMES: ` + 2. `SD_VERSION: ;` + + Or using comma separator: + 1. `ENV_NAMES: ` + 2. `SD_VERSION: ,` + + Or using space separator: + 1. `ENV_NAMES: ` + 2. `SD_VERSION: ` + + Or with explicit parameters (using newline): 1. `ENV_NAMES: ` 2. `SD_SOURCE_TYPE: artifact` 3. `SD_VERSION: \n` 4. `SD_REPO_MERGE_MODE: basic-merge` + + Or with explicit parameters (using semicolon): + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ;` + 4. `SD_REPO_MERGE_MODE: basic-merge` + + Or with explicit parameters (using comma): + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ,` + 4. `SD_REPO_MERGE_MODE: basic-merge` + + Or with explicit parameters (using space): + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ` + 4. `SD_REPO_MERGE_MODE: basic-merge` 2. GitHub Instance pipeline is started with parameters: 1. `ENV_NAMES: ` 2. `GH_ADDITIONAL_PARAMS: "SD_VERSION=\\n"` - Or with explicit parameters: + Or using comma separator: + 1. `ENV_NAMES: ` + 2. `GH_ADDITIONAL_PARAMS: "SD_VERSION=,"` + + Or using space separator: + 1. `ENV_NAMES: ` + 2. `GH_ADDITIONAL_PARAMS: "SD_VERSION= "` + + Or with explicit parameters (using newline): 1. `ENV_NAMES: ` 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION=\\n,SD_REPO_MERGE_MODE=basic-merge"` + Or with explicit parameters (using comma): + 1. `ENV_NAMES: ` + 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION=,,SD_REPO_MERGE_MODE=basic-merge"` + + Or with explicit parameters (using space): + 1. `ENV_NAMES: ` + 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION= ,SD_REPO_MERGE_MODE=basic-merge"` + **Steps:** 1. The `process_sd` job runs in the pipeline: @@ -420,17 +512,43 @@ The SD processing logic depends on: **Trigger:** > [!Note] -> One of the following conditions must be met: +> One of the following conditions must be met. 1. GitLab Instance pipeline is started with parameters: 1. `ENV_NAMES: ` 2. `SD_SOURCE_TYPE: artifact` 3. `SD_VERSION: \n` 4. `SD_REPO_MERGE_MODE: basic-exclusion-merge` + + Or using semicolon separator: + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ;` + 4. `SD_REPO_MERGE_MODE: basic-exclusion-merge` + + Or using comma separator: + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ,` + 4. `SD_REPO_MERGE_MODE: basic-exclusion-merge` + + Or using space separator: + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ` + 4. `SD_REPO_MERGE_MODE: basic-exclusion-merge` 2. GitHub Instance pipeline is started with parameters: 1. `ENV_NAMES: ` 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION=\\n,SD_REPO_MERGE_MODE=basic-exclusion-merge"` + Or using comma separator: + 1. `ENV_NAMES: ` + 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION=,,SD_REPO_MERGE_MODE=basic-exclusion-merge"` + + Or using space separator: + 1. `ENV_NAMES: ` + 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION= ,SD_REPO_MERGE_MODE=basic-exclusion-merge"` + **Steps:** 1. The `process_sd` job runs in the pipeline: @@ -458,17 +576,43 @@ The SD processing logic depends on: **Trigger:** > [!Note] -> One of the following conditions must be met: +> One of the following conditions must be met. 1. GitLab Instance pipeline is started with parameters: 1. `ENV_NAMES: ` 2. `SD_SOURCE_TYPE: artifact` 3. `SD_VERSION: \n` 4. `SD_REPO_MERGE_MODE: basic-exclusion-merge` + + Or using semicolon separator: + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ;` + 4. `SD_REPO_MERGE_MODE: basic-exclusion-merge` + + Or using comma separator: + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ,` + 4. `SD_REPO_MERGE_MODE: basic-exclusion-merge` + + Or using space separator: + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ` + 4. `SD_REPO_MERGE_MODE: basic-exclusion-merge` 2. GitHub Instance pipeline is started with parameters: 1. `ENV_NAMES: ` 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION=\\n,SD_REPO_MERGE_MODE=basic-exclusion-merge"` + Or using comma separator: + 1. `ENV_NAMES: ` + 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION=,,SD_REPO_MERGE_MODE=basic-exclusion-merge"` + + Or using space separator: + 1. `ENV_NAMES: ` + 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION= ,SD_REPO_MERGE_MODE=basic-exclusion-merge"` + **Steps:** 1. The `process_sd` job runs in the pipeline: @@ -495,17 +639,43 @@ The SD processing logic depends on: **Trigger:** > [!Note] -> One of the following conditions must be met: +> One of the following conditions must be met. Multiple SDs with `extended-merge` mode are not supported and will result in an error. 1. GitLab Instance pipeline is started with parameters: 1. `ENV_NAMES: ` 2. `SD_SOURCE_TYPE: artifact` 3. `SD_VERSION: \n` 4. `SD_REPO_MERGE_MODE: extended-merge` + + Or using semicolon separator: + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ;` + 4. `SD_REPO_MERGE_MODE: extended-merge` + + Or using comma separator: + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ,` + 4. `SD_REPO_MERGE_MODE: extended-merge` + + Or using space separator: + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ` + 4. `SD_REPO_MERGE_MODE: extended-merge` 2. GitHub Instance pipeline is started with parameters: 1. `ENV_NAMES: ` 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION=\\n,SD_REPO_MERGE_MODE=extended-merge"` + Or using comma separator: + 1. `ENV_NAMES: ` + 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION=,,SD_REPO_MERGE_MODE=extended-merge"` + + Or using space separator: + 1. `ENV_NAMES: ` + 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION= ,SD_REPO_MERGE_MODE=extended-merge"` + **Steps:** 1. The `process_sd` job runs in the pipeline: @@ -534,17 +704,43 @@ The SD processing logic depends on: **Trigger:** > [!Note] -> One of the following conditions must be met: +> One of the following conditions must be met. Multiple SDs with `extended-merge` mode are not supported and will result in an error. 1. GitLab Instance pipeline is started with parameters: 1. `ENV_NAMES: ` 2. `SD_SOURCE_TYPE: artifact` 3. `SD_VERSION: \n` 4. `SD_REPO_MERGE_MODE: extended-merge` + + Or using semicolon separator: + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ;` + 4. `SD_REPO_MERGE_MODE: extended-merge` + + Or using comma separator: + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ,` + 4. `SD_REPO_MERGE_MODE: extended-merge` + + Or using space separator: + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ` + 4. `SD_REPO_MERGE_MODE: extended-merge` 2. GitHub Instance pipeline is started with parameters: 1. `ENV_NAMES: ` 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION=\\n,SD_REPO_MERGE_MODE=extended-merge"` + Or using comma separator: + 1. `ENV_NAMES: ` + 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION=,,SD_REPO_MERGE_MODE=extended-merge"` + + Or using space separator: + 1. `ENV_NAMES: ` + 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION= ,SD_REPO_MERGE_MODE=extended-merge"` + **Steps:** 1. The `process_sd` job runs in the pipeline: @@ -570,17 +766,43 @@ The SD processing logic depends on: **Trigger:** > [!Note] -> One of the following conditions must be met: +> One of the following conditions must be met. 1. GitLab Instance pipeline is started with parameters: 1. `ENV_NAMES: ` 2. `SD_SOURCE_TYPE: artifact` 3. `SD_VERSION: \n` 4. `SD_REPO_MERGE_MODE: replace` + + Or using semicolon separator: + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ;` + 4. `SD_REPO_MERGE_MODE: replace` + + Or using comma separator: + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ,` + 4. `SD_REPO_MERGE_MODE: replace` + + Or using space separator: + 1. `ENV_NAMES: ` + 2. `SD_SOURCE_TYPE: artifact` + 3. `SD_VERSION: ` + 4. `SD_REPO_MERGE_MODE: replace` 2. GitHub Instance pipeline is started with parameters: 1. `ENV_NAMES: ` 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION=\\n,SD_REPO_MERGE_MODE=replace"` + Or using comma separator: + 1. `ENV_NAMES: ` + 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION=,,SD_REPO_MERGE_MODE=replace"` + + Or using space separator: + 1. `ENV_NAMES: ` + 2. `GH_ADDITIONAL_PARAMS: "SD_SOURCE_TYPE=artifact,SD_VERSION= ,SD_REPO_MERGE_MODE=replace"` + **Steps:** 1. The `process_sd` job runs in the pipeline: diff --git a/docs/use-cases/system-certificate.md b/docs/use-cases/system-certificate.md new file mode 100644 index 000000000..4aaff192c --- /dev/null +++ b/docs/use-cases/system-certificate.md @@ -0,0 +1,3 @@ +# SSL Certificate Processing Use Cases + +This document describes use cases for SSL certificate processing. diff --git a/docs/use-cases/template-inheritance.md b/docs/use-cases/template-inheritance.md new file mode 100644 index 000000000..24524d497 --- /dev/null +++ b/docs/use-cases/template-inheritance.md @@ -0,0 +1,210 @@ +# Template Inheritance (Template Composition) Use Cases + +- [Template Inheritance (Template Composition) Use Cases](#template-inheritance-template-composition-use-cases) + - [Overview](#overview) + - [Parent Templates Download and Selection](#parent-templates-download-and-selection) + - [UC-TI-PT-1: Build child template using a single parent template](#uc-ti-pt-1-build-child-template-using-a-single-parent-template) + - [UC-TI-PT-2: Build child template composed from multiple parent templates](#uc-ti-pt-2-build-child-template-composed-from-multiple-parent-templates) + - [Composite Structure Selection](#composite-structure-selection) + - [UC-TI-CS-1: Use explicit `composite_structure` from child Template Descriptor](#uc-ti-cs-1-use-explicit-composite_structure-from-child-template-descriptor) + - [Overrides in Child Template](#overrides-in-child-template) + - [UC-TI-OV-1: Override parent parameters for Cloud template](#uc-ti-ov-1-override-parent-parameters-for-cloud-template) + - [UC-TI-OV-2: Override parent parameters for Namespace template](#uc-ti-ov-2-override-parent-parameters-for-namespace-template) + +## Overview + +This document describes use cases for [Template Inheritance] +The child template can inherit data from one or more parent templates. +The child template can also override selected parent values. + +Important: + +- `overrides-parent` is part of Template Inheritance logic. +- `template_override` is a different feature and is described in [Template Override](/docs/features/template-override.md). + +## Parent Templates Download and Selection + +This section explains how `parent-templates` and `parent` links are resolved when the child template is built. + +### UC-TI-PT-1: Build child template using a single parent template + +**Pre-requisites:** + +1. A child Template Repository exists and is configured to build an Environment Template artifact. +2. The child repository contains a Template Descriptor with a single parent template reference: + + ```yaml + parent-templates: + basic-cloud: basic-product-template:10.1.3 + tenant: + parent: basic-cloud + cloud: + parent: basic-cloud + namespaces: + - name: "{env}-billing" + parent: basic-cloud + ``` + +**Trigger:** + +Template repository pipeline is started to build the child template artifact. + +**Steps:** + +1. The build pipeline starts Template Inheritance processing. +2. The pipeline resolves `tenant.parent`, `cloud.parent`, and `namespaces[].parent` using alias `basic-cloud` from `parent-templates`. +3. The pipeline publishes a regular EnvGene child template artifact. + +**Results:** + +1. Child template artifact is created successfully. +2. Inherited components are taken from `basic-cloud` according to descriptor references. +3. Parent templates are resolved from `parent-templates` entries (`application:version`). + +### UC-TI-PT-2: Build child template composed from multiple parent templates + +**Pre-requisites:** + +1. Child Template Repository contains a Template Descriptor that references multiple parent templates and composes namespaces from them: + + ```yaml + parent-templates: + basic-cloud: basic-product-template:10.1.3 + default-oss: core-product-templates:1.5.3 + default-billing: billing-product-templates:3.7.12 + default-bss: bss-product-templates:2.0.0 + tenant: + parent: basic-cloud + cloud: "{{ templates_dir }}/env_templates/composite/cloud.yml.j2" + composite_structure: "{{ templates_dir }}/env_templates/composite/composite_structure.yml.j2" + namespaces: + - name: "{env}-oss" + parent: default-oss + - name: "{env}-billing" + parent: default-billing + - name: "{env}-bss" + parent: default-bss + ``` + +2. All referenced parent template artifacts are available and accessible to the pipeline. + +**Trigger:** + +Template repository pipeline is started to build the child template artifact. + +**Steps:** + +1. The build pipeline starts Template Inheritance processing. +2. The pipeline resolves `tenant.parent` and each `namespaces[].parent` by alias from `parent-templates`. +3. `cloud` and `composite_structure` are taken from child template paths defined in descriptor. +4. The pipeline publishes a regular EnvGene child template artifact. + +**Results:** + +1. Child template artifact is created successfully. +2. Tenant and namespaces are composed from referenced parent templates. +3. Cloud and composite structure are taken from child repository sources. + +## Composite Structure Selection + +This section explains how explicit `composite_structure` is selected and used. + +### UC-TI-CS-1: Use explicit `composite_structure` from child Template Descriptor + +**Pre-requisites:** + +1. Child Template Descriptor specifies `composite_structure` explicitly: + + ```yaml + composite_structure: "{{ templates_dir }}/env_templates/composite/composite_structure.yml.j2" + ``` + +2. The referenced file exists in the child repository template sources. + +**Trigger:** + +Template repository pipeline is started to build the child template artifact. + +**Steps:** + +1. The build pipeline reads `composite_structure` path from child descriptor. +2. The path value is saved in built template artifact and later used during instance generation to render `composite_structure.yml`. + +**Results:** + +1. Built artifact contains the explicit child `composite_structure` reference. +2. User can verify generated instance contains `composite_structure.yml` rendered from that path. + +## Overrides in Child Template + +This section explains override logic in Template Inheritance. + +- Use `overrides-parent` when a child template inherits from a parent and changes selected fields. +- Do not mix this with `template_override`. `template_override` is applied in instance generation and is documented separately. + +### UC-TI-OV-1: Override parent parameters for Cloud template + +**Pre-requisites:** + +1. Child Template Descriptor defines `cloud.parent` and includes `cloud.overrides-parent` section: + + ```yaml + cloud: + parent: basic-template + overrides-parent: + deployParameters: + DEPLOY_PARAM1: "DEPLOY_PARAM1_VALUE" + e2eParameters: + E2E_PARAM1: "E2E_PARAM1_VALUE" + ``` + +2. The parent template `basic-template` exists and contains a Cloud template. + +**Trigger:** + +Template repository pipeline is started to build the child template artifact. + +**Steps:** + +1. The pipeline loads Cloud template from parent `basic-template`. +2. The pipeline applies `cloud.overrides-parent` over inherited Cloud values. +3. The pipeline publishes a child artifact with merged Cloud template. + +**Results:** + +1. Child Cloud template is inherited from parent and then updated by `cloud.overrides-parent`. +2. User can verify only supported override sections are used: `profile`, parameter maps, and parameter sets. + +### UC-TI-OV-2: Override parent parameters for Namespace template + +**Pre-requisites:** + +1. Child Template Descriptor defines a namespace with `parent` and includes `overrides-parent` section: + + ```yaml + namespaces: + - name: "{env}-bss" + parent: default-bss + overrides-parent: + technicalConfigurationParameters: + TECH_CONF_PARAM1: "TECH_CONF_PARAM1_VALUE" + deployParameterSets: + - "bss-overrides" + ``` + +2. The parent template `default-bss` exists and contains the referenced namespace template. + +**Trigger:** + +Template repository pipeline is started to build the child template artifact. + +**Steps:** + +1. The pipeline loads Namespace template from parent `default-bss`. +2. The pipeline applies `namespaces[].overrides-parent` over inherited Namespace values. +3. The pipeline publishes a child artifact with merged Namespace template. + +**Results:** + +1. Child Namespace template is inherited from parent and then updated by `namespaces[].overrides-parent`. +2. User can verify overridden parameter maps and parameter sets are present in the produced template. diff --git a/github_workflows/instance-repo-pipeline/.github/README.md b/github_workflows/instance-repo-pipeline/.github/README.md new file mode 100644 index 000000000..fb3fb3625 --- /dev/null +++ b/github_workflows/instance-repo-pipeline/.github/README.md @@ -0,0 +1,750 @@ +# EnvGene GitHub Workflow + +
+ +User Guide + +[![GitHub Actions](https://img.shields.io/badge/GitHub_Actions-workflow_dispatch-2088FF?style=flat-square&logo=github-actions&logoColor=white)](https://docs.github.com/en/actions) +[![Manual Trigger](https://img.shields.io/badge/trigger-manual-orange?style=flat-square)](#how-to-trigger-the-workflow) + +
+ +![EnvGene Workflow](docs/assets/envgene-workflow-header.png) + +- [EnvGene GitHub Workflow](#envgene-github-workflow) + - [Overview](#overview) + - [Installation](#installation) + - [Prerequisites](#prerequisites) + - [Step 1: Copy the Pipeline](#step-1-copy-the-pipeline) + - [Step 2: Configure Required Secrets](#step-2-configure-required-secrets) + - [Step 3: Optional — Repository Variables](#step-3-optional--repository-variables) + - [Step 4: Optional — Customize Configuration](#step-4-optional--customize-configuration) + - [Verifying the Setup](#verifying-the-setup) + - [Quick Start](#quick-start) + - [Workflow Structure](#workflow-structure) + - [Pipeline Steps](#pipeline-steps) + - [Job: `process_environment_variables`](#job-process_environment_variables) + - [Job: `envgene_execution` (runs per environment in matrix)](#job-envgene_execution-runs-per-environment-in-matrix) + - [Workflow Dispatch Inputs](#workflow-dispatch-inputs) + - [Understanding the 10-Input Limit](#understanding-the-10-input-limit) + - [Input Reference](#input-reference) + - [GH\_ADDITIONAL\_PARAMS — Passing Extra Parameters](#gh_additional_params--passing-extra-parameters) + - [What Is GH\_ADDITIONAL\_PARAMS?](#what-is-gh_additional_params) + - [Format and Syntax](#format-and-syntax) + - [Examples](#examples) + - [JSON Values and Escaping](#json-values-and-escaping) + - [When to Use pipeline\_vars.env Instead](#when-to-use-pipeline_varsenv-instead) + - [Adding New Parameters](#adding-new-parameters) + - [Option A: Add as Workflow Input (If Under the Limit)](#option-a-add-as-workflow-input-if-under-the-limit) + - [Option B: Use GH\_ADDITIONAL\_PARAMS](#option-b-use-gh_additional_params) + - [Option C: Use pipeline\_vars.env](#option-c-use-pipeline_varsenv) + - [Adding New Jobs and Conditional Execution](#adding-new-jobs-and-conditional-execution) + - [Step 1: Ensure the Variable Is Available](#step-1-ensure-the-variable-is-available) + - [Step 2: Expose the Variable as a Job Output](#step-2-expose-the-variable-as-a-job-output) + - [Step 3: Add the Job Step with an if Condition](#step-3-add-the-job-step-with-an-if-condition) + - [Complete Example: Adding a Custom Job](#complete-example-adding-a-custom-job) + - [Parameter Priority](#parameter-priority) + - [Repository Variables (vars)](#repository-variables-vars) + - [Variables Used by the Workflow](#variables-used-by-the-workflow) + - [How to Add Repository Variables](#how-to-add-repository-variables) + - [When Variables Are Empty or Missing](#when-variables-are-empty-or-missing) + - [Adding Custom Variables](#adding-custom-variables) + - [Using Different Docker Registries](#using-different-docker-registries) + - [GitHub Container Registry (GHCR)](#github-container-registry-ghcr) + - [Google Artifact Registry (GAR)](#google-artifact-registry-gar) + - [How to Trigger the Workflow](#how-to-trigger-the-workflow) + - [Via GitHub Actions UI](#via-github-actions-ui) + - [Via GitHub API](#via-github-api) + - [Directory Structure](#directory-structure) + - [Use Case Scenarios](#use-case-scenarios) + - [Scenario 1: Full Deployment (Environment Build + Effective Set)](#scenario-1-full-deployment-environment-build--effective-set) + - [Scenario 2: Environment Build Only (No Effective Set)](#scenario-2-environment-build-only-no-effective-set) + - [Scenario 3: Update Template Version and Rebuild](#scenario-3-update-template-version-and-rebuild) + - [Scenario 4: Blue-Green Operation](#scenario-4-blue-green-operation) + - [Scenario 5: Credential Rotation](#scenario-5-credential-rotation) + - [Scenario 6: Process Solution Descriptor from Artifact](#scenario-6-process-solution-descriptor-from-artifact) + - [Scenario 7: Generate New Environment Inventory](#scenario-7-generate-new-environment-inventory) + - [Scenario 8: Multiple Environments in One Run](#scenario-8-multiple-environments-in-one-run) + - [Further Reading](#further-reading) + +## Overview + +The **EnvGene** workflow (`Envgene.yml`) is a GitHub Actions pipeline that automates environment generation, configuration, and deployment for the EnvGene platform. It provides the same functionality as the GitLab-based instance pipeline, adapted for GitHub Actions. + +> [!NOTE] +> The workflow is **manually triggered only** (`workflow_dispatch`). There is no automatic trigger on push or pull request. + +The workflow supports: + +- Environment inventory generation +- Application and registry definition processing +- Solution Descriptor (SD) processing +- Environment build +- Effective Set generation +- Blue-Green management +- Credential rotation +- Git commit of generated artifacts + +--- + +## Installation + +This section describes what you need to set up the EnvGene workflow in your instance repository. + +### Prerequisites + +- A GitHub repository (instance repository) with the [EnvGene instance structure](/docs/samples/instance-repository/) +- GitHub Actions enabled for the repository +- GitHub-hosted runners (or self-hosted runners with Docker available) + +### Step 1: Copy the Pipeline + +Copy the `.github` directory from this folder to the root of your instance repository: + +```bash +cp -r github_workflows/instance-repo-pipeline/.github /path/to/your/instance-repo/ +``` + +The copied structure includes the workflow, scripts, configuration files, and the `load-env-files` action. + +### Step 2: Configure Required Secrets + +Go to **Settings** → **Secrets and variables** → **Actions** → **Secrets**, and add: + +| Secret | Required | Description | +|---------------------------|-------------------|---------------------------------------------------------------------| +| `SECRET_KEY` | When using Fernet | Fernet key for credential encryption | +| `ENVGENE_AGE_PUBLIC_KEY` | When using SOPS | Public key from EnvGene AGE key pair (SOPS encryption) | +| `ENVGENE_AGE_PRIVATE_KEY` | When using SOPS | Private key from EnvGene AGE key pair (SOPS decryption) | +| `GH_ACCESS_TOKEN` | Yes | GitHub token with `contents: write` to commit changes to repository | +| `GCP_SA_KEY` | When using GAR | Full JSON key of GCP service account for Artifact Registry access | + +> [!NOTE] +> At least one encryption method (Fernet or SOPS) must be configured if your repository uses encrypted credentials. See [Credential Encryption](/docs/how-to/credential-encryption.md) for details. + +### Step 3: Optional — Repository Variables + +Configure variables in **Settings** → **Secrets and variables** → **Actions** → **Variables** to override defaults: + +| Variable | Default | Purpose | +|----------------------------|----------------------|-----------------------------------------------| +| `DOCKER_REGISTRY` | `ghcr.io/netcracker` | Docker registry for EnvGene images | +| `GH_RUNNER_TAG_NAME` | `ubuntu-22.04` | Runner label for workflow jobs | +| `GH_RUNNER_SCRIPT_TIMEOUT` | `10` | Job timeout in minutes | + +See [Repository Variables (vars)](#repository-variables-vars) for details. For a step-by-step guide on GHCR and GAR, see [Using Different Docker Registries in Envgene.yml](/docs/how-to/docker-registry-configuration.md). + +### Step 4: Optional — Customize Configuration + +- **`.github/pipeline_vars.env`** — Optional overrides loaded by the workflow (for example debugging or recurring runs). Leave empty or add variables as needed. Default pipeline settings live in the `env:` block of `.github/workflows/Envgene.yml`. + +### Verifying the Setup + +1. Ensure the workflow file is at `.github/workflows/Envgene.yml`. +1. Ensure required secrets are set. +1. Trigger the workflow manually (see [Quick Start](#quick-start)) with a valid `ENV_NAMES` value. + +For initializing a new instance repository from scratch, see [Environment Instance Repository Installation Guide](/docs/how-to/envgene-maitanance.md). + +## Quick Start + +> [!TIP] +> New to EnvGene? Start with [Installation](#installation), then come back here. + +1. Ensure the pipeline is installed (see [Installation](#installation)). +1. Go to **Actions** → **EnvGene Execution** → **Run workflow**. +1. Fill in **ENV_NAMES** (e.g. `cluster-01/env-01`) and any other parameters. +1. Click **Run workflow**. + +## Workflow Structure + +The workflow consists of two main jobs: + +| Job | Purpose | +|---------------------------------|-----------------------------------------------------------------| +| `process_environment_variables` | Parses inputs, loads config, builds matrix, exports variables | +| `envgene_execution` | Runs EnvGene steps per environment (matrix job) | + +The first job prepares all parameters and passes them to the second job via a shared `.env` artifact and job outputs. The second job runs once per environment in the matrix. + +### Pipeline Steps + +The following sections describe each step in the pipeline as defined in `Envgene.yml`. Steps marked as *conditional* run only when their condition is met. + +#### Job: `process_environment_variables` + +| Step | Description | +|---------------------------------|--------------------------------------------------------------| +| Repository Checkout | Checks out the repository (without persisting credentials) | +| Load environment variables | Loads `pipeline_vars.env` into `GITHUB_ENV` | +| Process Input Parameters | Exports workflow inputs to environment | +| Process additional variables | Parses `GH_ADDITIONAL_PARAMS` and adds to environment | +| Create env_generation_params | Builds `ENV_GENERATION_PARAMS` JSON from SD/ENV variables | +| Multiple Environment Processing | Generates environment matrix from `ENV_NAMES` | +| Create .env file | Dumps all environment variables to `.env` | +| Upload .env as artifact | Uploads `.env` for use by `envgene_execution` job | + +#### Job: `envgene_execution` (runs per environment in matrix) + +| Step | Condition | Description | +|--------------------------------|---------------------------------------------------------------------------------|--------------------------------------------------------------| +| Repository Checkout | Always | Checks out repository with full history | +| Download environment-file | Always | Downloads `.env` artifact from previous job | +| Prepare environment | Always | Restores env vars, sets `PACKAGE_NAME`, extracts cluster/env | +| Create name for dynamic secret | Always | Sets `SECRET_NAME` for cluster-specific secrets | +| Create env file for container | Always | Exports env to `.env.container` for Docker steps | +| **BG_MANAGE** | `BG_MANAGE == 'true'` | Blue-Green operations: state management, validation | +| **ENV_INVENTORY_GENERATION** | One of: `ENV_INVENTORY_CONTENT`, `ENV_SPECIFIC_PARAMS`, `ENV_TEMPLATE_NAME` set | Generates Environment Inventory | +| **CREDENTIAL_ROTATION** | `CRED_ROTATION_PAYLOAD` not empty | Rotates credentials per payload | +| **APP_REG_DEF_PROCESS** | `ENV_BUILDER == 'true'` | Sets template version, renders App/Reg definitions | +| **PROCESS_SD** | `SD_SOURCE_TYPE` + `SD_DATA` or `SD_VERSION` | Processes Solution Descriptor | +| **ENV_BUILD** | `ENV_BUILDER == 'true'` | Generates Environment Instance from templates | +| **GENERATE_EFFECTIVE_SET** | `GENERATE_EFFECTIVE_SET == 'true'` | Generates Effective Set (SBOMs, validation, artifacts) | +| **GIT_COMMIT** | Always | Commits changes to repository | + +Each conditional step (in **bold**) also uploads its output as an artifact. The `GIT_COMMIT` step always runs at the end of the pipeline. + +## Workflow Dispatch Inputs + +### Understanding the 10-Input Limit + +> [!IMPORTANT] +> GitHub Actions limits `workflow_dispatch` to **10 input parameters**. The EnvGene pipeline uses 9 of them for the most common parameters. The 10th slot is reserved for `GH_ADDITIONAL_PARAMS`, which acts as a container for all other parameters. + +This design lets you pass any number of additional parameters without hitting the limit. + +### Input Reference + +| Input | Required | Default | Type | Description | +|--------------------------|----------|-----------|--------|--------------------------------------------------------| +| `ENV_NAMES` | Yes | — | string | Environment(s) to process. Format: `cluster/env` | +| `DEPLOYMENT_TICKET_ID` | No | `""` | string | Ticket ID used as commit message prefix | +| `ENV_TEMPLATE_VERSION` | No | `""` | string | Template version to apply (e.g. `env-template:v1.2.3`) | +| `ENV_BUILDER` | No | `"true"` | choice | Enable environment build | +| `GENERATE_EFFECTIVE_SET` | No | `"false"` | choice | Enable Effective Set generation | +| `GET_PASSPORT` | No | `"false"` | choice | Enable Cloud Passport discovery | +| `CMDB_IMPORT` | No | `"false"` | choice | Enable CMDB export | +| `GH_ADDITIONAL_PARAMS` | No | `""` | string | Comma-separated key-value pairs for other parameters | + +For full parameter semantics, see [Instance Pipeline Parameters](/docs/instance-pipeline-parameters.md). + +## GH_ADDITIONAL_PARAMS — Passing Extra Parameters + +### What Is GH_ADDITIONAL_PARAMS? + +`GH_ADDITIONAL_PARAMS` is a single string input that carries all pipeline parameters that are not exposed as separate workflow inputs. It is parsed by `.github/scripts/process_additional_variables.sh`, which adds each `KEY=VALUE` pair to the workflow environment. + +Use it for parameters such as: + +- `BG_MANAGE`, `BG_STATE` — Blue-Green operations +- `SD_SOURCE_TYPE`, `SD_VERSION`, `SD_DATA` — Solution Descriptor +- `ENV_SPECIFIC_PARAMS`, `ENV_TEMPLATE_NAME` — Environment configuration +- `EFFECTIVE_SET_CONFIG` — Effective Set options +- `CRED_ROTATION_PAYLOAD` — Credential rotation +- Any other parameter from [Instance Pipeline Parameters](/docs/instance-pipeline-parameters.md) + +### Format and Syntax + +**Format:** `KEY1=VALUE1,KEY2=VALUE2,KEY3=VALUE3` + +**Rules:** + +- Pairs are separated by commas. +- Each pair is `KEY=VALUE` (no spaces around `=`). +- Keys and values are trimmed of leading/trailing whitespace. +- Empty pairs are ignored. + +### Examples + +**Simple values:** + +```text +BG_MANAGE=true,SD_SOURCE_TYPE=artifact,SD_VERSION=my-app:v1.0 +``` + +**With JSON (escape double quotes):** + +```text +EFFECTIVE_SET_CONFIG={\"version\": \"v2.0\", \"app_chart_validation\": \"false\"} +``` + +**Multiple parameters:** + +```text +SD_SOURCE_TYPE=json,SD_DATA=[{\"version\":2.1,\"type\":\"solutionDeploy\"}],ENV_SPECIFIC_PARAMS={\"tenantName\":\"my-tenant\"} +``` + +**Blue-Green state:** + +```text +BG_MANAGE=true,BG_STATE={\"controllerNamespace\":\"bss-controller\",\"originNamespace\":{\"name\":\"bss-origin\",\"state\":\"active\"}} +``` + +### JSON Values and Escaping + +For JSON values: + +1. Escape internal double quotes: `\"` instead of `"`. +1. Be aware that commas inside JSON are used as pair separators. If your JSON contains commas, the parser may split it incorrectly. + +> [!CAUTION] +> **Workaround for complex JSON:** Use `pipeline_vars.env` (see below) or pass the parameter via the GitHub API with proper escaping. Commas inside JSON values may cause incorrect parsing. + +### When to Use pipeline_vars.env Instead + +Use `.github/pipeline_vars.env` when: + +- You have complex JSON with many commas. +- You want to keep sensitive or long values out of the UI. +- You need the same values across many runs (e.g. for debugging). + +Variables in `pipeline_vars.env` must be in standard `KEY=VALUE` format. Do **not** wrap them in `GH_ADDITIONAL_PARAMS`. + +## Adding New Parameters + +### Option A: Add as Workflow Input (If Under the Limit) + +If you have fewer than 10 inputs and want a dedicated UI field: + +1. Add the input under `on.workflow_dispatch.inputs` in `Envgene.yml`: + +```yaml +on: + workflow_dispatch: + inputs: + # ... existing inputs ... + MY_NEW_PARAM: + required: false + default: "" + type: string + description: "Description of the parameter" +``` + +1. Add a line in the "Process Input Parameters" step to export it: + +```yaml +echo "MY_NEW_PARAM=${{ github.event.inputs.MY_NEW_PARAM }}" >> $GITHUB_ENV +``` + +1. If the parameter controls job execution, add it to `process_environment_variables.outputs` (see [Adding New Jobs](#adding-new-jobs-and-conditional-execution)). + +### Option B: Use GH_ADDITIONAL_PARAMS + +1. Pass the parameter in `GH_ADDITIONAL_PARAMS`, e.g. `MY_NEW_PARAM=value`. +1. It will be parsed and added to `GITHUB_ENV` automatically. +1. If you need it for conditional steps, add it to the job outputs (see below). + +### Option C: Use pipeline_vars.env + +1. Add the variable to `.github/pipeline_vars.env`: + +```text +MY_NEW_PARAM=my_value +``` + +1. It will be loaded by the `load-env-files` action. +1. If you need it for conditional steps, add it to the job outputs. + +## Adding New Jobs and Conditional Execution + +To add a new step that runs only when a parameter is set, follow these steps. + +### Step 1: Ensure the Variable Is Available + +The variable must be present in `GITHUB_ENV` after the `process_environment_variables` job. It can come from: + +- A workflow input (and the "Process Input Parameters" step) +- `GH_ADDITIONAL_PARAMS` (parsed by `process_additional_variables.sh`) +- `pipeline_vars.env` (loaded by `load-env-files`) + +### Step 2: Expose the Variable as a Job Output + +Add the variable to the `outputs` of `process_environment_variables` in `Envgene.yml`: + +```yaml +jobs: + process_environment_variables: + outputs: + env_matrix: ${{ steps.matrix-generator.outputs.env_matrix }} + # ... existing outputs ... + MY_NEW_FEATURE: ${{ env.MY_NEW_FEATURE }} +``` + +Without this, the next job cannot use it in `if` conditions. + +### Step 3: Add the Job Step with an if Condition + +Add your step inside the `envgene_execution` job with an `if`: + +```yaml +- name: MY_NEW_JOB + if: needs.process_environment_variables.outputs.MY_NEW_FEATURE == 'true' + run: | + # Your commands here +``` + +**Common condition patterns:** + +| Condition type | Example | +|---------------------|-----------------------------------------------------------------| +| Equals string | `needs.process_environment_variables.outputs.MY_VAR == 'true'` | +| Not empty | `needs.process_environment_variables.outputs.MY_VAR != ''` | +| Logical OR | `(condition1) \|\| (condition2)` | +| Logical AND | `(condition1) && (condition2)` | +| Multiple conditions | `outputs.ENV_BUILDER == 'true' && outputs.SD_VERSION != ''` | + +### Complete Example: Adding a Custom Job + +Assume you want a step that runs only when `RUN_CUSTOM_VALIDATION=true`. + +**1. Pass the parameter** via `GH_ADDITIONAL_PARAMS`: + +```text +RUN_CUSTOM_VALIDATION=true +``` + +**2. Add the output** in `Envgene.yml`: + +```yaml +process_environment_variables: + outputs: + # ... existing ... + RUN_CUSTOM_VALIDATION: ${{ env.RUN_CUSTOM_VALIDATION }} +``` + +**3. Add the step** in `envgene_execution`: + +```yaml +- name: CUSTOM_VALIDATION + if: needs.process_environment_variables.outputs.RUN_CUSTOM_VALIDATION == 'true' + run: | + echo "Running custom validation..." + # Your validation logic +``` + +## Parameter Priority + +When the same parameter is set in multiple places, the effective value is chosen by this order (highest first): + +1. Workflow input parameters (UI or API) +1. `pipeline_vars.env` +1. Repository variables (`vars`) +1. Organization variables + +## Repository Variables (vars) + +Repository variables are configured in **Settings → Secrets and variables → Actions → Variables** (repository-level) or at the organization level. They are referenced in the workflow as `vars.VARIABLE_NAME` and are available to all workflow runs. + +### Variables Used by the Workflow + +| Variable | Purpose | Default when empty | +|----------------------------------|------------------------------------------------|----------------------| +| `DOCKER_REGISTRY` | Docker registry base for EnvGene images | `ghcr.io/netcracker` | +| `DOCKER_CLOUD_REGISTRY_PROVIDER` | Cloud provider for registry auth (GCP for GAR) | (empty) | +| `GH_RUNNER_TAG_NAME` | Runner label for jobs (e.g. ubuntu-22.04) | `ubuntu-22.04` | +| `GH_RUNNER_SCRIPT_TIMEOUT` | Job timeout in minutes | `10` | + +### How to Add Repository Variables + +1. Go to your repository on GitHub. +1. Open **Settings** → **Secrets and variables** → **Actions**. +1. Open the **Variables** tab. +1. Click **New repository variable**. +1. Enter the name (e.g. `DOCKER_REGISTRY`) and value. +1. Click **Add variable**. + +### When Variables Are Empty or Missing + +The workflow uses fallback values when a variable is not set or is empty. For example: + +```yaml +runs-on: ${{ vars.GH_RUNNER_TAG_NAME || 'ubuntu-22.04' }} +DOCKER_IMAGE_NAME_ENVGENE: "${{ vars.DOCKER_REGISTRY || 'ghcr.io/netcracker' }}/qubership-envgene" +timeout-minutes: ${{ fromJSON(vars.GH_RUNNER_SCRIPT_TIMEOUT || '10') }} +``` + +- If `vars.GH_RUNNER_TAG_NAME` is empty or missing → `ubuntu-22.04` is used. +- If `vars.DOCKER_REGISTRY` is empty or missing → `ghcr.io/netcracker` is used. +- If `vars.GH_RUNNER_SCRIPT_TIMEOUT` is empty or missing → `10` is used. + +> [!TIP] +> You do not need to define these variables for the workflow to run; defaults are applied automatically. + +### Adding Custom Variables + +To use your own variables in the workflow: + +1. Add the variable in **Settings → Secrets and variables → Actions → Variables**. +1. Reference it in `Envgene.yml` as `${{ vars.MY_CUSTOM_VAR }}`. +1. For optional variables with a default, use: `${{ vars.MY_CUSTOM_VAR || 'default_value' }}`. + +For a full list of supported repository variables, see [EnvGene Repository Variables](/docs/envgene-repository-variables.md). + +## Using Different Docker Registries + +The workflow pulls EnvGene Docker images (envgene, pipegene, effective-set-generator) from a registry. By default, images are pulled from GitHub Container Registry (GHCR). You can switch to another registry such as Google Artifact Registry (GAR) by configuring the appropriate variables and secrets. + +### GitHub Container Registry (GHCR) + +GHCR is the default registry. No additional configuration is required. + +| Where to configure | Parameter | Value | +|--------------------------|-------------------|--------------------------------| +| **Settings → Variables** | `DOCKER_REGISTRY` | `ghcr.io/netcracker` (default) | + +**Authentication:** GitHub Actions automatically authenticates to `ghcr.io` using `GITHUB_TOKEN` when pulling images. No extra secrets are needed. + +**Image names:** The workflow builds image paths as `$DOCKER_REGISTRY/qubership-envgene`, `$DOCKER_REGISTRY/qubership-pipegene`, etc. For GHCR, the full path is `ghcr.io/netcracker/qubership-envgene:1.31.18` (see `DOCKER_IMAGE_TAG_*` in `.github/workflows/Envgene.yml`). + +### Google Artifact Registry (GAR) + +To use Google Artifact Registry, configure the registry URL and GCP authentication. + +| Where to configure | Parameter | Value | +|--------------------------|----------------------------------|----------------------------------------------| +| **Settings → Variables** | `DOCKER_REGISTRY` | `REGION-docker.pkg.dev/PROJECT_ID/REPO_NAME` | +| **Settings → Variables** | `DOCKER_CLOUD_REGISTRY_PROVIDER` | `GCP` | +| **Settings → Secrets** | `GCP_SA_KEY` | Full JSON key of the GCP service account | + +**Example `DOCKER_REGISTRY` for GAR:** + +```text +europe-west1-docker.pkg.dev/my-gcp-project/envgene-images +``` + +**Authentication:** When `DOCKER_CLOUD_REGISTRY_PROVIDER` is set to `GCP`, the workflow runs a step that authenticates to GAR using `docker login` with the `_json_key` method. The `GCP_SA_KEY` secret must contain the full JSON key of a GCP service account that has `Artifact Registry Reader` (or equivalent) permissions. + +**How to set up GCP_SA_KEY:** + +1. Create a GCP service account with access to your Artifact Registry repository. +1. Create a JSON key for the service account (IAM → Service Accounts → Keys → Add Key). +1. Copy the entire JSON content. +1. In GitHub: **Settings → Secrets and variables → Actions → Secrets** → **New repository secret**. +1. Name: `GCP_SA_KEY`, Value: paste the full JSON. + +> [!IMPORTANT] +> The service account must have at least `Artifact Registry Reader` role on the repository. For private images, ensure the key is not expired and has the correct permissions. + +**Summary:** + +| Parameter | Location | Required for GAR | +|----------------------------------|-----------|---------------------------| +| `DOCKER_REGISTRY` | Variables | Yes - full GAR path | +| `DOCKER_CLOUD_REGISTRY_PROVIDER` | Variables | Yes - set to `GCP` | +| `GCP_SA_KEY` | Secrets | Yes - JSON key content | + +## How to Trigger the Workflow + +### Via GitHub Actions UI + +1. Open your repository on GitHub. +1. Go to **Actions**. +1. Select **EnvGene Execution**. +1. Click **Run workflow**. +1. Choose the branch, fill in parameters, and run. + +### Via GitHub API + +
+Click to expand API example + +```bash +curl -X POST \ + -H "Authorization: token " \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos///actions/workflows/Envgene.yml/dispatches \ + -d '{ + "ref": "main", + "inputs": { + "ENV_NAMES": "cluster-01/env-01", + "ENV_BUILDER": "true", + "GENERATE_EFFECTIVE_SET": "true", + "DEPLOYMENT_TICKET_ID": "QBSHP-0001", + "GH_ADDITIONAL_PARAMS": "EFFECTIVE_SET_CONFIG={\"version\": \"v2.0\", \"app_chart_validation\": \"false\"}" + } + }' +``` + +Replace ``, ``, ``, and `main` as needed. + +
+ +## Directory Structure + +```text +instance-repo-pipeline/ +└── .github/ + ├── README.md # This guide (EnvGene GitHub workflow) + ├── docs/ + │ └── assets/ + │ └── envgene-workflow-header.png + ├── actions/ + │ └── load-env-files/ # Loads .env files into GITHUB_ENV + ├── scripts/ + │ ├── generate_env_matrix.sh # Builds environment matrix from ENV_NAMES + │ ├── process_additional_variables.sh # Parses GH_ADDITIONAL_PARAMS + │ ├── process_matrix_iteration.sh # Extracts cluster/env from matrix + │ └── create_env_generation_params.sh # Builds ENV_GENERATION_PARAMS JSON + ├── workflows/ + │ └── Envgene.yml # Main workflow definition + └── pipeline_vars.env # Optional overrides (template, often empty) +``` + +--- + +## Use Case Scenarios + +This section shows typical scenarios with example parameters and what happens when you run the workflow. + +### Scenario 1: Full Deployment (Environment Build + Effective Set) + +**Goal:** Build the environment and generate the Effective Set for deployment. + +| Parameter | Value | +|--------------------------|--------------------------| +| `ENV_NAMES` | `prod-cluster/prod-01` | +| `ENV_BUILDER` | `true` | +| `GENERATE_EFFECTIVE_SET` | `true` | +| `DEPLOYMENT_TICKET_ID` | `QBSHP-1234` | + +**Steps that run:** APP_REG_DEF_PROCESS → ENV_BUILD → GENERATE_EFFECTIVE_SET → GIT_COMMIT + +**Result:** Environment Instance is generated, Effective Set is created in `environments/prod-cluster/prod-01/effective-set/`, changes are committed to the repository. + +--- + +### Scenario 2: Environment Build Only (No Effective Set) + +**Goal:** Regenerate the Environment Instance without generating the Effective Set (e.g. for validation or template updates). + +| Parameter | Value | +|----------------|------------------------| +| `ENV_NAMES` | `dev-cluster/dev-01` | +| `ENV_BUILDER` | `true` | + +**Steps that run:** APP_REG_DEF_PROCESS → ENV_BUILD → GIT_COMMIT + +**Result:** Environment Instance is regenerated and committed. GENERATE_EFFECTIVE_SET is skipped. + +--- + +### Scenario 3: Update Template Version and Rebuild + +**Goal:** Switch to a new template version and rebuild the environment. + +| Parameter | Value | +|------------------------|------------------------| +| `ENV_NAMES` | `prod-cluster/prod-01` | +| `ENV_BUILDER` | `true` | +| `ENV_TEMPLATE_VERSION` | `env-template:v2.1.0` | + +**Steps that run:** APP_REG_DEF_PROCESS (updates template version) → ENV_BUILD → GIT_COMMIT + +**Result:** `env_definition.yml` is updated with the new template version, environment is rebuilt with the new template, changes are committed. + +--- + +### Scenario 4: Blue-Green Operation + +**Goal:** Perform a Blue-Green operation (e.g. warmup, state change). + +| Parameter | Value | +|------------------------|-------------------------------------| +| `ENV_NAMES` | `prod-cluster/prod-01` | +| `GH_ADDITIONAL_PARAMS` | `BG_MANAGE=true,BG_STATE={...}` | + +**Example `GH_ADDITIONAL_PARAMS` value:** + +```text +BG_MANAGE=true,BG_STATE={\"controllerNamespace\":\"bss-ctrl\",\"originNamespace\":{\"name\":\"bss-origin\",\"state\":\"ACTIVE\",\"version\":\"v1.0\"},\"peerNamespace\":{\"name\":\"bss-peer\",\"state\":\"CANDIDATE\",\"version\":\"v1.1\"},\"updateTime\":\"2024-01-15T10:00:00Z\"} +``` + +**Steps that run:** BG_MANAGE → GIT_COMMIT + +**Result:** BG state is validated, state files are updated in the repository, namespace objects are copied if warmup. No ENV_BUILD or Effective Set. + +--- + +### Scenario 5: Credential Rotation + +**Goal:** Rotate credentials for an environment without rebuilding. + +| Parameter | Value | +|------------------------|-----------------------------------------| +| `ENV_NAMES` | `prod-cluster/prod-01` | +| `GH_ADDITIONAL_PARAMS` | `CRED_ROTATION_PAYLOAD={...}` | + +**Example `GH_ADDITIONAL_PARAMS` value:** + +```text +CRED_ROTATION_PAYLOAD={\"credentials\":[{\"name\":\"db-password\",\"newValue\":\"\"}]} +``` + +**Steps that run:** CREDENTIAL_ROTATION → GIT_COMMIT + +**Result:** Credentials are updated per payload, changes are committed. See [Credential Rotation](/docs/features/cred-rotation.md) for full payload format. + +--- + +### Scenario 6: Process Solution Descriptor from Artifact + +**Goal:** Fetch SD from an artifact and merge it into the repository. + +| Parameter | Value | +|------------------------|-----------------------------------------------------------------| +| `ENV_NAMES` | `prod-cluster/prod-01` | +| `GH_ADDITIONAL_PARAMS` | `SD_SOURCE_TYPE=artifact,SD_VERSION=my-solution:v1.2.3,...` | + +**Steps that run:** PROCESS_SD → GIT_COMMIT + +**Result:** SD is downloaded from the artifact registry, merged (or replaced) into `environments/prod-cluster/prod-01/Inventory/solution-descriptor/sd.yaml`, committed. + +--- + +### Scenario 7: Generate New Environment Inventory + +**Goal:** Create a new Environment Inventory (`env_definition.yml`) for a new environment. + +| Parameter | Value | +|------------------------|-----------------------------------------------------------------| +| `ENV_NAMES` | `new-cluster/new-env` | +| `GH_ADDITIONAL_PARAMS` | `ENV_INVENTORY_INIT=true,ENV_TEMPLATE_NAME=my-env-template` | + +**Steps that run:** ENV_INVENTORY_GENERATION → GIT_COMMIT + +**Result:** New `env_definition.yml` is created at `environments/new-cluster/new-env/Inventory/`, committed. See [Environment Inventory Generation](/docs/features/env-inventory-generation.md). + +--- + +### Scenario 8: Multiple Environments in One Run + +**Goal:** Process several environments with the same parameters. + +| Parameter | Value | +|---------------|--------------------------------------------| +| `ENV_NAMES` | `cluster-01/env-01,cluster-01/env-02,...` | +| `ENV_BUILDER` | `true` | + +**Steps that run:** For each environment in the matrix: APP_REG_DEF_PROCESS → ENV_BUILD → GIT_COMMIT (parallel jobs) + +**Result:** Three separate `envgene_execution` jobs run in parallel, each processes one environment. All changes are committed in a single workflow run. + +--- + +## Further Reading + +| Document | Description | +|---------------------------------------------------------------------------------------|--------------------------------| +| [Instance Pipeline Parameters](/docs/instance-pipeline-parameters.md) | Full parameter reference | +| [EnvGene Pipelines](/docs/envgene-pipelines.md) | Pipeline flow and descriptions | +| [Using Different Docker Registries](/docs/how-to/docker-registry-configuration.md) | GHCR and GAR configuration | +| [Blue-Green Deployment](/docs/features/blue-green-deployment.md) | BG-related parameters | +| [SD Processing](/docs/use-cases/sd-processing.md) | Solution Descriptor use cases | + +--- + +
+ +EnvGene GitHub Workflow — Part of the Qubership EnvGene platform + +
diff --git a/github_workflows/instance-repo-pipeline/.github/configuration/config.env b/github_workflows/instance-repo-pipeline/.github/configuration/config.env deleted file mode 100644 index 8bc9db08b..000000000 --- a/github_workflows/instance-repo-pipeline/.github/configuration/config.env +++ /dev/null @@ -1,14 +0,0 @@ -CI_PROJECT_DIR=/workspace -INSTANCES_DIR=/workspace/environments -COMMIT_ENV=true -GIT_STRATEGY=none -GITHUB_USER_EMAIL=envgene@qubership.org -GITHUB_USER_NAME=envgene -PROJECT_DIR=/workspace -SECRET_POSTFIX=custom_secret -envgen_args=-vvv -envgen_debug=true -module_ansible_cfg=/module/ansible/ansible.cfg -module_ansible_dir=/module/ansible -module_config_default=/module/templates/defaults.yaml -module_inventory=/workspace/configuration/inventory.yaml diff --git a/github_workflows/instance-repo-pipeline/.github/docs/README.md b/github_workflows/instance-repo-pipeline/.github/docs/README.md deleted file mode 100644 index 06fc64c26..000000000 --- a/github_workflows/instance-repo-pipeline/.github/docs/README.md +++ /dev/null @@ -1,133 +0,0 @@ -# EnvGene GitHub Pipeline Usage Guide - -- [EnvGene GitHub Pipeline Usage Guide](#envgene-github-pipeline-usage-guide) - - [Overview](#overview) - - [Available Parameters](#available-parameters) - - [How to Trigger the Pipeline](#how-to-trigger-the-pipeline) - - [Via GitHub Actions UI](#via-github-actions-ui) - - [Via GitHub API Call](#via-github-api-call) - - [Parameter Priority](#parameter-priority) - - [`pipeline_vars.env`](#pipeline_varsenv) - - [Pipeline Customization](#pipeline-customization) - - [How to Get the Pipeline](#how-to-get-the-pipeline) - -## Overview - -The EnvGene pipeline (`Envgene.yaml`) is a GitHub Actions workflow that supports both manual UI triggers and API calls. It provides the same full EnvGene functionality as the GitLab pipeline. - -## Available Parameters - -GitHub's UI limits manual inputs to 10 parameters. To handle this limitation while maintaining full functionality, we expose the most frequently used parameters directly in the UI and group the remaining parameters within the [GH_ADDITIONAL_PARAMS](https://github.com/Netcracker/qubership-envgene/blob/main/docs/instance-pipeline-parameters.md#gh_additional_params) parameter. - -Only a limited number of core parameters are available in the GitHub version of the pipeline: - -- [ENV_NAMES](https://github.com/Netcracker/qubership-envgene/blob/main/docs/instance-pipeline-parameters.md#env_names) -- [DEPLOYMENT_TICKET_ID](https://github.com/Netcracker/qubership-envgene/blob/main/docs/instance-pipeline-parameters.md#deployment_ticket_id) -- [ENV_TEMPLATE_VERSION](https://github.com/Netcracker/qubership-envgene/blob/main/docs/instance-pipeline-parameters.md#env_template_version) -- [ENV_BUILDER](https://github.com/Netcracker/qubership-envgene/blob/main/docs/instance-pipeline-parameters.md#env_builder) -- [GENERATE_EFFECTIVE_SET](https://github.com/Netcracker/qubership-envgene/blob/main/docs/instance-pipeline-parameters.md#generate_effective_set) -- [GET_PASSPORT](https://github.com/Netcracker/qubership-envgene/blob/main/docs/instance-pipeline-parameters.md#get_passport) -- [CMDB_IMPORT](https://github.com/Netcracker/qubership-envgene/blob/main/docs/instance-pipeline-parameters.md#cmdb_import) -- [GH_ADDITIONAL_PARAMS](https://github.com/Netcracker/qubership-envgene/blob/main/docs/instance-pipeline-parameters.md#gh_additional_params) - -The [GH_ADDITIONAL_PARAMS](https://github.com/Netcracker/qubership-envgene/blob/main/docs/instance-pipeline-parameters.md#gh_additional_params) parameter serves as a wrapper for all parameters except those listed above. This approach enables the transmission of all [Instance Pipeline parameters](https://github.com/Netcracker/qubership-envgene/blob/main/docs/instance-pipeline-parameters.md). - -## How to Trigger the Pipeline - -The pipeline can be triggered in two ways: - -### Via GitHub Actions UI - -1. Go to your GitHub repository -2. Navigate to the **Actions** tab -3. Select the **EnvGene Execution** workflow -4. Click **Run workflow** -5. Fill in the required parameters -6. Click **Run workflow** - -### Via GitHub API Call - -Use the GitHub API to trigger the workflow: - -```bash -curl -X POST \ - -H "Authorization: token " \ - -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos///actions/workflows/Envgene.yaml/dispatches \ - -d '{ - "ref": "", - "inputs": { - "": "", - "GH_ADDITIONAL_PARAMS": "KEY1=VALUE1,KEY2=VALUE2" - } - }' -``` - -**Example:** - -```bash -curl -X POST \ - -H "Authorization: token token-placeholder-123" \ - -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos/qubership/instance-repo/actions/workflows/Envgene.yaml/dispatches \ - -d '{ - "ref": "main", - "inputs": { - "ENV_NAMES": "test-cluster/e01", - "ENV_BUILDER": "true", - "GENERATE_EFFECTIVE_SET": "true", - "DEPLOYMENT_TICKET_ID": "QBSHP-0001", - "GH_ADDITIONAL_PARAMS": "EFFECTIVE_SET_CONFIG={\"version\": \"v2.0\", \"app_chart_validation\": \"false\"}" - } - }' -``` - -## Parameter Priority - -The pipeline allows parameters to be defined in multiple locations, listed in descending order of priority: - -1. Pipeline execution input parameters -2. `pipeline_vars.env` file -3. Repository or repository group CI/CD variables - -Together, these form the complete context used by EnvGene. -When parameter keys from different sources overlap, their values are replaced according to the priority order specified above. - -Best practices for setting variables are: - -- Define in CI/CD variables [EnvGene repository variables](https://github.com/Netcracker/qubership-envgene/blob/main/docs/envgene-repository-variables.md) -- Pass through GitHub Actions UI or GitHub API Call [Instance Pipeline parameters](https://github.com/Netcracker/qubership-envgene/blob/main/docs/instance-pipeline-parameters.md) -- Use `pipeline_vars.env` for debug purposes - -## `pipeline_vars.env` - -Variables in this file must be described in Dotenv format and follow POSIX shell variable syntax. - -For example: - -```text -ENV_SPECIFIC_PARAMS={"clusterParams":{"clusterEndpoint":"","clusterToken":""},"additionalTemplateVariables":{"":""},"cloudName":"","envSpecificParamsets":{"":["paramsetA"],"cloud":["paramsetB"]},"paramsets":{"paramsetA":{"version":"","name":"","parameters":{"":""},"applications":[{"appName":"","parameters":{"":""}}]},"paramsetB":{"version":"","name":"","parameters":{"":""},"applications":[]}},"credentials":{"credX":{"type":"","data":{"username":"","password":""}},"credY":{"type":"","data":{"secret":""}}}} -EFFECTIVE_SET_CONFIG={\"version\": \"v2.0\", \"app_chart_validation\": \"false\"} -``` - -Variables set in this file must NOT be wrapped with [GH_ADDITIONAL_PARAMS](https://github.com/Netcracker/qubership-envgene/blob/main/docs/instance-pipeline-parameters.md#gh_additional_params) - -## Pipeline Customization - -Pipeline customization is only possible in a limited number of cases: - -- **Reducing** the number of input parameters in the UI -- Changing default values of input parameters in the UI -- Changing the mandatory status of input parameters in the UI - -All these changes are made and described in the `Envgene.yaml` section: - -```yaml -on: - workflow_dispatch: - inputs: -``` - -## How to Get the Pipeline - -To get the pipeline, you need to copy the [.github](/github_workflows/instance-repo-pipeline/.github) directory to the root of your GitHub repository diff --git a/github_workflows/instance-repo-pipeline/.github/docs/assets/envgene-workflow-header.png b/github_workflows/instance-repo-pipeline/.github/docs/assets/envgene-workflow-header.png new file mode 100644 index 000000000..cbd93dc86 Binary files /dev/null and b/github_workflows/instance-repo-pipeline/.github/docs/assets/envgene-workflow-header.png differ diff --git a/github_workflows/instance-repo-pipeline/.github/workflows/Envgene.yml b/github_workflows/instance-repo-pipeline/.github/workflows/Envgene.yml index bf9146cec..1a9596a7d 100644 --- a/github_workflows/instance-repo-pipeline/.github/workflows/Envgene.yml +++ b/github_workflows/instance-repo-pipeline/.github/workflows/Envgene.yml @@ -56,12 +56,18 @@ on: default: "" env: + #CONFIGURATION + GITHUB_USER_EMAIL: envgene@qubership.org + GITHUB_USER_NAME: envgene + SECRET_POSTFIX: custom_secret CI_COMMIT_REF_NAME: ${{ github.ref_name }} CI_PROJECT_DIR: /workspace + INSTANCES_DIR: /workspace/environments SECRET_KEY: ${{ secrets.SECRET_KEY }} ENVGENE_AGE_PUBLIC_KEY: ${{ secrets.ENVGENE_AGE_PUBLIC_KEY }} ENVGENE_AGE_PRIVATE_KEY: ${{ secrets.ENVGENE_AGE_PRIVATE_KEY }} GITHUB_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} + DOCKER_CLOUD_REGISTRY_PROVIDER: ${{ vars.DOCKER_CLOUD_REGISTRY_PROVIDER || '' }} #DOCKER_IMAGE_NAMES DOCKER_IMAGE_NAME_PIPEGENE: "${{ vars.DOCKER_REGISTRY || 'ghcr.io/netcracker' }}/qubership-pipegene" @@ -69,9 +75,9 @@ env: DOCKER_IMAGE_NAME_EFFECTIVE_SET_GENERATOR: "${{ vars.DOCKER_REGISTRY || 'ghcr.io/netcracker' }}/qubership-effective-set-generator" #DOCKER_IMAGE_TAGS - DOCKER_IMAGE_TAG_PIPEGENE: "1.21.2" - DOCKER_IMAGE_TAG_ENVGENE: "1.21.2" - DOCKER_IMAGE_TAG_EFFECTIVE_SET_GENERATOR: "1.21.2" + DOCKER_IMAGE_TAG_PIPEGENE: "1.32.6" + DOCKER_IMAGE_TAG_ENVGENE: "1.32.6" + DOCKER_IMAGE_TAG_EFFECTIVE_SET_GENERATOR: "1.32.6" jobs: process_environment_variables: @@ -81,13 +87,17 @@ jobs: outputs: env_matrix: ${{ steps.matrix-generator.outputs.env_matrix }} BG_MANAGE: ${{ env.BG_MANAGE }} - ENV_TEMPLATE_TEST: ${{ env.ENV_TEMPLATE_TEST }} - ENV_SPECIFIC_PARAMS: ${{ env.ENV_SPECIFIC_PARAMS }} - ENV_TEMPLATE_NAME: ${{ env.ENV_TEMPLATE_NAME }} + ENV_INVENTORY_CONTENT: ${{ env.ENV_INVENTORY_CONTENT }} CRED_ROTATION_PAYLOAD: ${{ env.CRED_ROTATION_PAYLOAD }} ENV_BUILDER: ${{ env.ENV_BUILDER }} GENERATE_EFFECTIVE_SET: ${{ env.GENERATE_EFFECTIVE_SET }} ENV_INVENTORY_INIT: ${{ env.ENV_INVENTORY_INIT }} + ENV_SPECIFIC_PARAMS: ${{ env.ENV_SPECIFIC_PARAMS }} + ENV_TEMPLATE_NAME: ${{ env.ENV_TEMPLATE_NAME }} + SD_SOURCE_TYPE: ${{ env.SD_SOURCE_TYPE }} + SD_DATA: ${{ env.SD_DATA }} + SD_VERSION: ${{ env.SD_VERSION }} + DOCKER_CLOUD_REGISTRY_PROVIDER: ${{ env.DOCKER_CLOUD_REGISTRY_PROVIDER }} steps: - name: Repository Checkout uses: actions/checkout@v4.1.0 @@ -98,7 +108,6 @@ jobs: uses: ./.github/actions/load-env-files with: paths: | - .github/configuration/config.env .github/pipeline_vars.env - name: Process Input Parameters @@ -106,6 +115,7 @@ jobs: echo "DEPLOYMENT_TICKET_ID=${{ github.event.inputs.DEPLOYMENT_TICKET_ID }}" >> $GITHUB_ENV echo "ENV_NAMES=${{ github.event.inputs.ENV_NAMES }}" >> $GITHUB_ENV echo "ENV_BUILDER=${{ github.event.inputs.ENV_BUILDER }}" >> $GITHUB_ENV + echo "GENERATE_EFFECTIVE_SET=${{ github.event.inputs.GENERATE_EFFECTIVE_SET }}" >> $GITHUB_ENV echo "GET_PASSPORT=${{ github.event.inputs.GET_PASSPORT }}" >> $GITHUB_ENV echo "CMDB_IMPORT=${{ github.event.inputs.CMDB_IMPORT }}" >> $GITHUB_ENV echo "ENV_TEMPLATE_VERSION=${{ github.event.inputs.ENV_TEMPLATE_VERSION }}" >> $GITHUB_ENV @@ -155,6 +165,7 @@ jobs: with: fetch-depth: 0 + ### ENV VARS PROCESSING - START ### - name: Download environment-file uses: actions/download-artifact@v4 with: @@ -183,7 +194,17 @@ jobs: - name: Create env file for container run: env > .env.container - ### BG MANAGE ### + - name: Authenticate to GAR (Google Artifact Registry) + if: needs.process_environment_variables.outputs.DOCKER_CLOUD_REGISTRY_PROVIDER == 'GCP' + uses: docker/login-action@v4 + with: + registry: ${{ vars.DOCKER_REGISTRY }} + username: _json_key + password: ${{ secrets.GCP_SA_KEY }} + ### ENV VARS PROCESSING - END ### + + + ### BG MANAGE - START ### - name: BG_MANAGE if: needs.process_environment_variables.outputs.BG_MANAGE == 'true' env: @@ -192,6 +213,7 @@ jobs: docker run --rm -v ${{ github.workspace }}:${{ env.CI_PROJECT_DIR }} -w ${{ env.CI_PROJECT_DIR }} -e GITHUB_TOKEN -e FULL_ENV_NAME -e CI_PROJECT_DIR --user root --env-file .env.container --user root -e HOME=/root \ ${{ env.DOCKER_IMAGE_NAME_ENVGENE }}:${{ env.DOCKER_IMAGE_TAG_ENVGENE }} \ bash -c " + set -eo pipefail source /module/venv/bin/activate echo 'Executing BG Manage...' @@ -208,16 +230,17 @@ jobs: name: bg_manage_${{ env.PACKAGE_NAME }} path: environments/${{ matrix.environment }} include-hidden-files: true - ########################## + ### BG MANAGE - END ### - ### ENV_INVENTORY_GENERATION ### + ### ENV_INVENTORY_GENERATION - START ### - name: ENV_INVENTORY_GENERATION - if: needs.process_environment_variables.outputs.ENV_TEMPLATE_TEST == 'false' && (needs.process_environment_variables.outputs.ENV_SPECIFIC_PARAMS != '' || needs.process_environment_variables.outputs.ENV_TEMPLATE_NAME != '') + if: (needs.process_environment_variables.outputs.ENV_INVENTORY_CONTENT != '') || (needs.process_environment_variables.outputs.ENV_SPECIFIC_PARAMS != '' || needs.process_environment_variables.outputs.ENV_TEMPLATE_NAME != '') run: | docker run --rm -v ${{ github.workspace }}:${{ env.CI_PROJECT_DIR }} -w ${{ env.CI_PROJECT_DIR }} --env-file .env.container --user root -e HOME=/root \ ${{ env.DOCKER_IMAGE_NAME_ENVGENE }}:${{ env.DOCKER_IMAGE_TAG_ENVGENE }} \ bash -c " + set -eo pipefail source /module/venv/bin/activate echo 'Executing inventory generation...' @@ -229,21 +252,22 @@ jobs: - name: ENV_INVENTORY_GENERATION - Upload ENV_INVENTORY_GENERATION Package uses: actions/upload-artifact@v4 - if: needs.process_environment_variables.outputs.ENV_TEMPLATE_TEST == 'false' && (needs.process_environment_variables.outputs.ENV_SPECIFIC_PARAMS != '' || needs.process_environment_variables.outputs.ENV_TEMPLATE_NAME != '') + if: (needs.process_environment_variables.outputs.ENV_INVENTORY_CONTENT != '') || (needs.process_environment_variables.outputs.ENV_SPECIFIC_PARAMS != '' || needs.process_environment_variables.outputs.ENV_TEMPLATE_NAME != '') with: name: env_inventory_generation_${{ env.PACKAGE_NAME }} path: environments/${{ matrix.environment }} include-hidden-files: true - ########################## + ### ENV_INVENTORY_GENERATION - END ### - ### CREDENTIAL ROTATION ### + ### CREDENTIAL ROTATION - START ### - name: CREDENTIAL_ROTATION if: needs.process_environment_variables.outputs.CRED_ROTATION_PAYLOAD != '' run: | docker run --rm -v ${{ github.workspace }}:${{ env.CI_PROJECT_DIR }} -w ${{ env.CI_PROJECT_DIR }} --env-file .env.container --user root -e HOME=/root \ ${{ env.DOCKER_IMAGE_NAME_ENVGENE }}:${{ env.DOCKER_IMAGE_TAG_ENVGENE }} \ bash -c " + set -eo pipefail source /module/venv/bin/activate echo 'Executing credential rotation...' @@ -260,20 +284,21 @@ jobs: name: credential_rotation_${{ env.PACKAGE_NAME }} path: environments/${{ matrix.environment }} include-hidden-files: true - ########################## + ### CREDENTIAL ROTATION - END ### - ### APP_REG_DEF_PROCESS ### + ### APP_REG_DEF_PROCESS - START ### - name: APP_REG_DEF_PROCESS if: needs.process_environment_variables.outputs.ENV_BUILDER == 'true' run: | docker run --rm -v ${{ github.workspace }}:${{ env.CI_PROJECT_DIR }} -w ${{ env.CI_PROJECT_DIR }} --env-file .env.container --user root -e HOME=/root \ ${{ env.DOCKER_IMAGE_NAME_ENVGENE }}:${{ env.DOCKER_IMAGE_TAG_ENVGENE }} \ bash -c " + set -eo pipefail source /module/venv/bin/activate echo 'Executing APP_REG_DEF_PROCESS...' python3 /module/scripts/utils/log_pipe_params.py - /module/scripts/handle_certs.sh + /module/scripts/utils/handle_certs.sh python3 /build_env/scripts/build_env/env_template/set_template_version.py python3 /build_env/scripts/build_env/appregdef_render.py " @@ -288,26 +313,56 @@ jobs: configuration tmp include-hidden-files: true - ########################## + ### APP_REG_DEF_PROCESS - END ### - ### ENV_BUILD ### + ### PROCESS_SD - START ### + - name: PROCESS_SD + if: (needs.process_environment_variables.outputs.SD_SOURCE_TYPE == 'json' && needs.process_environment_variables.outputs.SD_DATA != '') || (needs.process_environment_variables.outputs.SD_SOURCE_TYPE == 'artifact' && needs.process_environment_variables.outputs.SD_VERSION != '') + run: | + docker run --rm -v ${{ github.workspace }}:${{ env.CI_PROJECT_DIR }} -w ${{ env.CI_PROJECT_DIR }} --env-file .env.container --user root -e HOME=/root \ + ${{ env.DOCKER_IMAGE_NAME_EFFECTIVE_SET_GENERATOR }}:${{ env.DOCKER_IMAGE_TAG_EFFECTIVE_SET_GENERATOR }} \ + bash -c " + set -eo pipefail + source /module/venv/bin/activate + + echo 'Executing PROCESS_SD...' + /module/scripts/utils/handle_certs.sh + base_env_path=\"\${CI_PROJECT_DIR}/environments/${{ matrix.environment }}\" + app_defs_path=\"\${base_env_path}/AppDefs\" + reg_defs_path=\"\${base_env_path}/RegDefs\" + if [ -n \"\$APP_REG_DEFS_JOB\" ] && [ -n \"\$APP_DEFS_PATH\" ]; then mkdir -p \"\$app_defs_path\" && cp -rf \"\$APP_DEFS_PATH\"/* \"\$app_defs_path\"; fi + if [ -n \"\$APP_REG_DEFS_JOB\" ] && [ -n \"\$REG_DEFS_PATH\" ]; then mkdir -p \"\$reg_defs_path\" && cp -fr \"\$REG_DEFS_PATH\"/* \"\$reg_defs_path\"; fi + if ! python3 /build_env/scripts/build_env/process_sd.py; then + echo 'PROCESS_SD failed with exit code \$?' + exit 1 + fi + " + + - name: PROCESS_SD - Upload PROCESS_SD Package + uses: actions/upload-artifact@v4 + if: (needs.process_environment_variables.outputs.SD_SOURCE_TYPE == 'json' && needs.process_environment_variables.outputs.SD_DATA != '') || (needs.process_environment_variables.outputs.SD_SOURCE_TYPE == 'artifact' && needs.process_environment_variables.outputs.SD_VERSION != '') + with: + name: process_sd_${{ env.PACKAGE_NAME }} + path: environments/${{ matrix.environment }} + include-hidden-files: true + ### PROCESS_SD - END ### + + + ### ENV_BUILD - START ### - name: ENV_BUILD if: needs.process_environment_variables.outputs.ENV_BUILDER == 'true' run: | - docker run --rm -v ${{ github.workspace }}:${{ env.CI_PROJECT_DIR }} -w ${{ env.CI_PROJECT_DIR }} --env-file .env.container --user root -e HOME=/root \ + docker run --rm -v ${{ github.workspace }}:${{ env.CI_PROJECT_DIR }} -w ${{ env.CI_PROJECT_DIR }} -e COMMIT_ENV=true --env-file .env.container --user root -e HOME=/root \ ${{ env.DOCKER_IMAGE_NAME_ENVGENE }}:${{ env.DOCKER_IMAGE_TAG_ENVGENE }} \ bash -c " + set -eo pipefail source /module/venv/bin/activate echo 'Executing ENV_BUILD...' python /module/scripts/utils/log_pipe_params.py - /module/scripts/handle_certs.sh - cd /build_env; python3 /build_env/scripts/build_env/main.py - - if [ \"\$ENV_TEMPLATE_TEST\" == \"true\" ]; then - env_name=\$(cat set_variable.txt) - sed -i \"s|\\\\\\\"envgeneNullValue\\\\\\\"|\\\\\\\"test_value\\\\\\\"|g\" \"\${CI_PROJECT_DIR}/environments/\$env_name/Credentials/credentials.yml\" - fi + /module/scripts/utils/handle_certs.sh + cd /build_env + python3 /build_env/scripts/build_env/main.py " - name: ENV_BUILD - Upload Build Env Package @@ -320,31 +375,57 @@ jobs: configuration tmp include-hidden-files: true - ########################## + ### ENV_BUILD - END ### - ### GENERATE EFFECTIVE SET ### + ### GENERATE EFFECTIVE SET - START ### - name: GENERATE_EFFECTIVE_SET if: needs.process_environment_variables.outputs.GENERATE_EFFECTIVE_SET == 'true' run: | docker run --rm -v ${{ github.workspace }}:${{ env.CI_PROJECT_DIR }} -w ${{ env.CI_PROJECT_DIR }} --env-file .env.container --user root -e HOME=/root \ ${{ env.DOCKER_IMAGE_NAME_EFFECTIVE_SET_GENERATOR }}:${{ env.DOCKER_IMAGE_TAG_EFFECTIVE_SET_GENERATOR }} \ bash -c " - # Export all variables using unified exporter - /module/scripts/handle_certs.sh - - echo \"Executing effective set generation...\" - if ! /module/scripts/prepare.sh \"generate_effective_set.yaml\"; then - echo \"Effective set generation failed with exit code \$?\" + set -eo pipefail + echo 'Executing effective set generation...' + source /module/venv/bin/activate + /module/scripts/utils/handle_certs.sh + mkdir -p \"\${CI_PROJECT_DIR}/configuration/certs\" + [ -f /default_cert.pem ] && cp /default_cert.pem \"\${CI_PROJECT_DIR}/configuration/certs/default_cert.pem\" || true + for cert in \"\${CI_PROJECT_DIR}/configuration/certs/\"*; do [ -f \"\$cert\" ] && keytool -import -trustcacerts -alias \"\$(basename \"\$cert\")\" -file \"\$cert\" -keystore /etc/ssl/certs/keystore.jks -storepass changeit -noprompt; done + if ! python3 /module/scripts/main.py decrypt_cred_files; then + echo 'decrypt_cred_files failed with exit code '\$? exit 1 fi - - env_path=\$(find \"\${CI_PROJECT_DIR}/environments\" -type d -name \"\$env_name\") - for path in \$env_path; do - if [ -d \"\$path/Credentials\" ]; then - chmod ugo+rw \"\$path/Credentials/\"* + base_env_path=\"\${CI_PROJECT_DIR}/environments/${{ matrix.environment }}\" + app_defs_path=\"\${base_env_path}/AppDefs\" + reg_defs_path=\"\${base_env_path}/RegDefs\" + if [ -n \"\$APP_REG_DEFS_JOB\" ] && [ -n \"\$APP_DEFS_PATH\" ]; then mkdir -p \"\$app_defs_path\" && cp -rf \"\$APP_DEFS_PATH\"/* \"\$app_defs_path\"; fi + if [ -n \"\$APP_REG_DEFS_JOB\" ] && [ -n \"\$REG_DEFS_PATH\" ]; then mkdir -p \"\$reg_defs_path\" && cp -fr \"\$REG_DEFS_PATH\"/* \"\$reg_defs_path\"; fi + if ! python3 /module/scripts/main.py validate_creds; then + echo 'validate_creds failed with exit code '\$? + exit 1 + fi + java_cmd=\"/deployments/run-java.sh --env-id=${{ matrix.environment }} --envs-path=\${CI_PROJECT_DIR}/environments --output=\${CI_PROJECT_DIR}/environments/${{ matrix.environment }}/effective-set --registries=\${CI_PROJECT_DIR}/configuration/registry.yml --sboms-path=\${CI_PROJECT_DIR}/sboms --sd-path=\${CI_PROJECT_DIR}/environments/${{ matrix.environment }}/Inventory/solution-descriptor/sd.yaml\" + if [ -n \"\$EFFECTIVE_SET_CONFIG\" ]; then + if ! python3 /module/scripts/handle_effective_set_config.py --effective-set-config \"\$EFFECTIVE_SET_CONFIG\"; then + echo 'handle_effective_set_config failed with exit code '\$? + exit 1 fi - done + extra_args=\$(jq -r '.extra_args // [] | join(\" \")' /tmp/effective_set_output.json) + java_cmd=\"\$java_cmd \$extra_args\" + fi + deploy_id=\"\${DEPLOYMENT_SESSION_ID:-\$DEPLOYMENT_TICKET_ID}\" + if [ -n \"\$deploy_id\" ]; then java_cmd=\"\$java_cmd --extra_params=DEPLOYMENT_SESSION_ID=\$deploy_id\"; fi + if ! eval \"\$java_cmd\"; then + echo 'Effective set generation failed with exit code '\$? + exit 1 + fi + if ! python3 /module/scripts/main.py encrypt_cred_files; then + echo 'encrypt_cred_files failed with exit code '\$? + exit 1 + fi + env_path=\"\${CI_PROJECT_DIR}/environments/${{ matrix.environment }}\" + if [ -d \"\$env_path/Credentials\" ]; then chmod -R ugo+rw \"\$env_path/Credentials\"; fi " - name: GENERATE_EFFECTIVE_SET - Upload Generate Effective Set Package @@ -352,30 +433,28 @@ jobs: if: needs.process_environment_variables.outputs.GENERATE_EFFECTIVE_SET == 'true' with: name: generate_effective_set_${{ env.PACKAGE_NAME }} - path: environments/${{ matrix.environment }} + path: | + environments/${{ matrix.environment }} + sboms + configuration include-hidden-files: true - ########################## + ### GENERATE EFFECTIVE SET - END ### - ### GIT COMMIT ### + ### GIT COMMIT - START ### - name: GIT_COMMIT run: | - docker run --rm -v ${{ github.workspace }}:${{ env.CI_PROJECT_DIR }} -w ${{ env.CI_PROJECT_DIR }} --env-file .env.container --user root -e HOME=/root \ + docker run --rm -v ${{ github.workspace }}:${{ env.CI_PROJECT_DIR }} -w ${{ env.CI_PROJECT_DIR }} -e COMMIT_ENV=true --env-file .env.container --user root -e HOME=/root \ ${{ env.DOCKER_IMAGE_NAME_ENVGENE }}:${{ env.DOCKER_IMAGE_TAG_ENVGENE }} \ bash -c " + set -eo pipefail source /module/venv/bin/activate - echo 'Prepare git_commit job for \${ENVIRONMENT_NAME}...' - - /module/scripts/handle_certs.sh - - git config --global --add safe.directory \"\${CI_PROJECT_DIR}\" - + /module/scripts/utils/handle_certs.sh # Execute git commit with proper error handling echo 'Executing git commit operations...' - if ! /module/scripts/prepare.sh \"git_commit.yaml\"; then - echo 'Git commit operations failed with exit code \$?' - echo 'This could be due to authentication issues or other git errors' + if ! /module/scripts/git_commit.sh; then + echo 'git_commit.sh failed with exit code '\$? exit 1 fi @@ -385,8 +464,6 @@ jobs: chmod ugo+rw \"\$path/Credentials/\"* fi done - - cp -rf \${CI_PROJECT_DIR}/environments \${CI_PROJECT_DIR}/git_envs " env > .env.container @@ -394,6 +471,8 @@ jobs: uses: actions/upload-artifact@v4 with: name: git_commit_${{ env.PACKAGE_NAME }} - path: environments/${{ matrix.environment }} + path: | + environments/${{ matrix.environment }} + sboms include-hidden-files: true - ########################## + ### GIT COMMIT - END ### diff --git a/github_workflows/instance-repo-pipeline/Dockerfile b/github_workflows/instance-repo-pipeline/Dockerfile index 0f5e643c8..a1183d179 100644 --- a/github_workflows/instance-repo-pipeline/Dockerfile +++ b/github_workflows/instance-repo-pipeline/Dockerfile @@ -1,13 +1,22 @@ +# syntax=docker/dockerfile:1 #-----------Stage 1----------- -FROM alpine:3.19 AS initial +FROM ghcr.io/netcracker/qubership-envgene-base-modules:1.0.1 AS initial + WORKDIR /workspace + COPY github_workflows/instance-repo-pipeline/.github /opt/github -# Make all shell scripts executable -RUN find /opt/github/scripts -type f -name "*.sh" -exec chmod +x {} \; +COPY github_workflows/instance-repo-pipeline/extend_logic/scripts/ /opt/github/extend_logic/scripts/ + +RUN find /opt/github/scripts -type f -name "*.sh" -exec chmod +x {} \; \ + && find /opt/github/extend_logic/scripts -type f -name "*.py" -exec chmod +x {} \; #-----------Stage 2----------- -FROM scratch +FROM ghcr.io/netcracker/qubership-envgene-base-modules:1.0.1 + +LABEL org.opencontainers.image.description="Github Workflows - Envgene Instance pipeline" + COPY --from=initial /opt/github /opt/github + USER 1000 -CMD ["sh", "-c", "sleep infinity"] \ No newline at end of file +CMD ["sh", "-c", "sleep infinity"] diff --git a/github_workflows/instance-repo-pipeline/extend_logic/scripts/apply_envgene_patch.py b/github_workflows/instance-repo-pipeline/extend_logic/scripts/apply_envgene_patch.py new file mode 100755 index 000000000..cd327bdad --- /dev/null +++ b/github_workflows/instance-repo-pipeline/extend_logic/scripts/apply_envgene_patch.py @@ -0,0 +1,607 @@ +#!/usr/bin/env python3 +import os +import re +import shutil +import sys +import zipfile +from pathlib import Path + +try: + from ruamel.yaml import YAML +except ImportError: + print("Error: ruamel.yaml required. Add to Dockerfile: pip install ruamel.yaml", file=sys.stderr) + sys.exit(1) + + +def resolve_version_tag(): + """Tag used in the output path (folder or zip name; matches pipeline image version).""" + tag = ( + os.environ.get("DOCKER_IMAGE_TAG") + or os.environ.get("INSTANCE_REPO_PIPELINE_IMAGE_TAG") + or "latest" + ).strip() + return tag or "latest" + + +def sanitize_tag(tag: str) -> str: + """Filesystem-safe directory name; rejects path separators.""" + tag = (tag or "").strip() + if not tag: + return "latest" + if ".." in tag or "/" in tag or "\\" in tag: + raise ValueError( + f"Invalid DOCKER_IMAGE_TAG: {tag!r} (must not contain path separators)" + ) + safe = re.sub(r"[^a-zA-Z0-9._-]", "_", tag) + if not safe or not re.search(r"[0-9a-zA-Z]", safe): + return "latest" + return safe + + +# ========== MERGE ENV: add or replace variables in .env files ========== + +def do_merge_env(target_file, content): + """Add or replace KEY=value in .env file.""" + text = target_file.read_text(encoding="utf-8") + lines = text.splitlines(keepends=True) + + # Parse existing: key -> (line_index, full_line) + existing = {} + for i, line in enumerate(lines): + s = line.strip() + if s and not s.startswith("#") and "=" in s: + key = s.split("=", 1)[0].strip() + if key: + existing[key] = (i, line) + + to_add = {k: v for k, v in content.items() if k not in existing} + to_update = {k: v for k, v in content.items() if k in existing} + + if not to_add and not to_update: + return "skipped (no changes)" + + def format_env(key, val): + val_str = str(val).strip('"\'') + return f'{key}="{val_str}"' if " " in val_str or "\n" in val_str else f"{key}={val_str}" + + # Replace existing + for key, value in to_update.items(): + idx, _ = existing[key] + lines[idx] = format_env(key, value) + "\n" + + # Append new at end (no extra blank line) + if to_add: + new_lines = "".join(format_env(k, v) + "\n" for k, v in to_add.items()) + lines.append(new_lines) + + target_file.write_text("".join(lines), encoding="utf-8") + + parts = [] + if to_update: + parts.append(f"updated {list(to_update.keys())}") + if to_add: + parts.append(f"inserted {list(to_add.keys())}") + return "; ".join(parts) + + +# ========== MERGE YAML PATH: add keys to block at dotted path (no section needed) ========== + +def _find_block_by_path(lines, path_str): + """Find (line_index, content_indent) of block at path like jobs.process_environment_variables.outputs.""" + parts = path_str.split(".") + search_start = 0 + block_line = None + content_indent = 2 + prev_indent = -1 + + for part in parts: + found = None + for i in range(search_start, len(lines)): + line = lines[i] + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + line_indent = len(line) - len(line.lstrip()) + if ":" in stripped and not stripped.startswith("-"): + key = stripped.split(":")[0].strip() + if key == part and line_indent > prev_indent: + found = (i, line_indent) + search_start = i + 1 + break + if found is None: + return None, 2 + block_line, prev_indent = found + content_indent = prev_indent + 2 + + return block_line, content_indent + + +def do_merge_yaml_path(target_file, content, path_str): + """Merge content into YAML block at dotted path (e.g. jobs.process_environment_variables.outputs).""" + lines = target_file.read_text(encoding="utf-8").splitlines(keepends=True) + + block_line, block_indent = _find_block_by_path(lines, path_str) + if block_line is None: + raise ValueError(f"Block '{path_str}' not found") + + block_key_indent = block_indent - 2 + existing = {} + for i in range(block_line + 1, len(lines)): + line = lines[i] + line_indent = len(line) - len(line.lstrip()) + if line.strip() and line_indent <= block_key_indent: + break + s = line.strip() + if s and not s.startswith("#") and ":" in s and not s.startswith("-"): + key = s.split(":")[0].strip() + if key: + existing[key] = (i, line) + + to_add = {k: v for k, v in content.items() if k not in existing} + to_update = {k: v for k, v in content.items() if k in existing} + + if not to_add and not to_update: + return "skipped (no changes)" + + yaml = YAML() + yaml.preserve_quotes = True + yaml.width = 4096 + + for key, value in to_update.items(): + idx, old_line = existing[key] + indent = len(old_line) - len(old_line.lstrip()) + from io import StringIO + buf = StringIO() + yaml.dump({key: value}, buf) + formatted = buf.getvalue().rstrip().split("\n") + new_line = "".join(" " * indent + s + "\n" for s in formatted) + lines[idx] = new_line + + if to_add: + from io import StringIO + buf = StringIO() + yaml.dump(to_add, buf) + text = buf.getvalue().rstrip() + spaces = " " * block_indent + indented = "\n".join(spaces + (s.lstrip() if s.startswith(" ") else s) for s in text.split("\n")) + lines.insert(block_line + 1, indented + "\n") + + target_file.write_text("".join(lines), encoding="utf-8") + + parts = [] + if to_update: + parts.append(f"updated {list(to_update.keys())}") + if to_add: + parts.append(f"inserted {list(to_add.keys())}") + return "; ".join(parts) + + +# ========== MERGE: add or replace keys after comment #SECTION (YAML) ========== + +def do_merge(target_file, content, section): + lines = target_file.read_text(encoding="utf-8").splitlines(keepends=True) + + # Find the line with comment (e.g. #DOCKER_IMAGE_NAMES) + section_line = None + indent = 2 + for i, line in enumerate(lines): + text = line.strip() + if text.startswith("#") and section.upper() in text.upper(): + section_line = i + indent = len(line) - len(line.lstrip()) + break + + if section_line is None: + raise ValueError(f"Section #{section} not found") + + # Collect all existing keys in file: key -> (line_number, line) + existing = {} + for i, line in enumerate(lines): + text = line.strip() + if text and not text.startswith("#") and ":" in text and not text.startswith("-"): + key = text.split(":")[0].strip() + if key: + existing[key] = (i, line) + + # Split: what to add, what to replace + to_add = {k: v for k, v in content.items() if k not in existing} + to_update = {k: v for k, v in content.items() if k in existing} + + if not to_add and not to_update: + return "skipped (no changes)" + + yaml = YAML() + yaml.preserve_quotes = True + yaml.width = 4096 + + # Replace existing keys + for key, value in to_update.items(): + idx, old_line = existing[key] + spaces = " " * (len(old_line) - len(old_line.lstrip())) + from io import StringIO + buf = StringIO() + yaml.dump({key: value}, buf) + formatted = buf.getvalue().rstrip().split("\n") + new_line = "".join(spaces + s + "\n" for s in formatted) + lines[idx] = new_line + + # Insert new keys after section comment + if to_add: + from io import StringIO + buf = StringIO() + yaml.dump(to_add, buf) + text = buf.getvalue().rstrip() + spaces = " " * max(indent, 2) + indented = "\n".join(spaces + (s.lstrip() if s.startswith(" ") else s) for s in text.split("\n")) + lines.insert(section_line + 1, indented + "\n") + + target_file.write_text("".join(lines), encoding="utf-8") + + parts = [] + if to_update: + parts.append(f"updated {list(to_update.keys())}") + if to_add: + parts.append(f"inserted {list(to_add.keys())}") + return "; ".join(parts) + + +# ========== INSERT: insert block after/before ### SECTION ### or by step name ========== + +def find_insert_position(lines, section, after=True): + """Find position and indent of marker line. + after=True: insert after ### SECTION - END ### + after=False: insert before ### SECTION - START ### + Returns (insert_pos, base_indent).""" + marker = f"### {section} - {'END' if after else 'START'} ###" + for i, line in enumerate(lines): + if marker in line.strip(): + indent = len(line) - len(line.lstrip()) + # after: insert at i+1; before: insert at i + pos = i + 1 if after else i + return pos, indent + return None, 0 + + +def find_step_position(lines, step_name, after=True): + """Find insert position by step name (e.g. 'Create env file for container'). + after=True: insert after the step ends; after=False: insert before the step starts. + Step name matching is case-insensitive and supports partial match. + Returns (insert_pos, base_indent) or (None, 0).""" + step_indent = None + step_start = None + step_end = None + step_name_lower = step_name.lower().strip() + + for i, line in enumerate(lines): + stripped = line.strip() + # Match step: - name: "..." or - name: '...' or - name: ... + m = re.match(r'^-\s*name\s*:\s*(?:"([^"]*)"|\'([^\']*)\'|(\S.*?))\s*$', stripped) + if m: + name = (m.group(1) or m.group(2) or m.group(3) or "").strip() + if step_name_lower in name.lower() or name.lower() in step_name_lower: + line_indent = len(line) - len(line.lstrip()) + step_start = i + step_indent = line_indent + # Step continues until next step (- name:) at same indent + for j in range(i + 1, len(lines)): + next_line = lines[j] + next_stripped = next_line.strip() + if not next_stripped: + continue + next_indent = len(next_line) - len(next_line.lstrip()) + # Next step: "- name:" at same indent level + if (next_indent <= step_indent and next_stripped.startswith("-") and + re.match(r'^-\s*name\s*:', next_stripped)): + step_end = j + break + if step_end is None: + step_end = len(lines) + break + + if step_start is None: + return None, 0 + + base_indent = step_indent if step_indent is not None else 6 + if after: + return step_end, base_indent + return step_start, base_indent + + +def fix_indent(text, spaces=4): + lines = text.split("\n") + min_indent = min((len(s) - len(s.lstrip()) for s in lines if s.strip()), default=0) + result = [] + for s in lines: + if s.strip(): + stripped = s[min_indent:] if min_indent else s + result.append(" " * spaces + stripped) + else: + result.append("") + return "\n".join(result) + + +def _make_insertion(content, base_indent, pos, total_lines): + """Build insertion string with 2 empty lines before and after content.""" + content = fix_indent(content.rstrip(), spaces=max(base_indent, 4)) + return "\n\n" + content + "\n\n" + + +def do_insert(target_file, content, after_section=None, before_section=None, + after_step=None, before_step=None, skip_if=None): + """Insert content at position determined by section marker or step name. + Exactly one of after_section, before_section, after_step, before_step must be set.""" + text = target_file.read_text(encoding="utf-8") + if skip_if and skip_if in text: + return "skipped (already present)" + + lines = text.splitlines(keepends=True) + pos = None + base_indent = 0 + + if after_section: + pos, base_indent = find_insert_position(lines, after_section, after=True) + if pos is None: + raise ValueError(f"Marker '### {after_section} - END ###' not found") + elif before_section: + pos, base_indent = find_insert_position(lines, before_section, after=False) + if pos is None: + raise ValueError(f"Marker '### {before_section} - START ###' not found") + elif after_step: + pos, base_indent = find_step_position(lines, after_step, after=True) + if pos is None: + raise ValueError(f"Step '{after_step}' not found") + elif before_step: + pos, base_indent = find_step_position(lines, before_step, after=False) + if pos is None: + raise ValueError(f"Step '{before_step}' not found") + else: + raise ValueError("One of after_section, before_section, after_step, before_step required") + + insertion = _make_insertion(content, base_indent, pos, len(lines)) + lines.insert(pos, insertion) + target_file.write_text("".join(lines), encoding="utf-8") + return "inserted" + + +# ========== MAIN LOGIC ========== + +def apply_patch(patch_path, base_dir): + yaml = YAML() + yaml.preserve_quotes = True + yaml.width = 4096 + + with open(patch_path, encoding="utf-8") as f: + operations = yaml.load(f) + + if operations is None: + operations = [] + elif not isinstance(operations, list): + operations = [operations] + + if not operations: + return [] + + def resolve_target(target_file_str): + """Resolve target file path. If base_dir is output dir, map .github/ -> output_dir/.""" + if target_file_str.startswith(".github/") and base_dir != Path("."): + return base_dir / target_file_str[len(".github/"):] + return base_dir / target_file_str + + # Default target from first operation (for operations without own target_file) + default_target = None + for op in operations: + if op.get("target_file"): + default_target = resolve_target(op["target_file"]) + break + + if not default_target: + raise ValueError("target_file required in patch (e.g. .github/workflows/Envgene.yml)") + + result = [] + + for i, op in enumerate(operations): + action = op.get("action") + path_str = op.get("path") or op.get("target_file", "file") + # Each operation can have its own target_file + target_file = ( + resolve_target(op["target_file"]) if op.get("target_file") else default_target + ) + + if not target_file.is_file(): + raise FileNotFoundError(f"File not found: {target_file}") + + if not action: + print(f" [Skip] Operation {i+1}: missing action", file=sys.stderr) + continue + + if action == "merge": + section = op.get("section") + content = op.get("content") or {} + if not isinstance(content, dict): + print(f" [Skip] Operation {i+1}: content must be dict", file=sys.stderr) + continue + # .env files: merge without section (KEY=value format) + if not section and str(target_file).endswith(".env"): + msg = do_merge_env(target_file, content) + result.append(f"merge env {path_str} ({msg})") + elif section: + msg = do_merge(target_file, content, section) + result.append(f"merge {path_str} ({msg})") + elif path_str and path_str != "file": + # YAML merge by path (e.g. jobs.process_environment_variables.outputs) + msg = do_merge_yaml_path(target_file, content, path_str) + result.append(f"merge {path_str} ({msg})") + else: + print(f" [Skip] Operation {i+1}: merge requires section, path, or .env target", file=sys.stderr) + + elif action == "insert": + after_section = op.get("after_section") + before_section = op.get("before_section") + after_step = op.get("after_step") + before_step = op.get("before_step") + content = op.get("content") + skip_if = op.get("skip_if_present") + + anchor_count = sum(1 for x in (after_section, before_section, after_step, before_step) if x) + if anchor_count != 1: + print( + f" [Skip] Operation {i+1}: insert requires exactly one of " + "after_section, before_section, after_step, before_step", + file=sys.stderr + ) + continue + if content is None: + print(f" [Skip] Operation {i+1}: insert requires content", file=sys.stderr) + continue + if isinstance(content, dict): + from io import StringIO + buf = StringIO() + yaml.dump(content, buf) + content = buf.getvalue() + + msg = do_insert( + target_file, content, + after_section=after_section, before_section=before_section, + after_step=after_step, before_step=before_step, skip_if=skip_if + ) + anchor = after_section or before_section or after_step or before_step + if after_section: + anchor_type = "after_section" + elif before_section: + anchor_type = "before_section" + elif after_step: + anchor_type = "after_step" + else: + anchor_type = "before_step" + result.append(f"insert {anchor_type} '{anchor}' ({msg})") + + else: + print(f" [Skip] Operation {i+1}: unknown action '{action}'", file=sys.stderr) + + return result + + +def init_output_dir(output_root, source_dir): + """Remove output_root and .github, then copy source_dir into output_root.""" + output_path = Path(output_root) + source_path = Path(source_dir) + if not source_path.is_dir(): + raise FileNotFoundError(f"Source directory not found: {source_path}") + for path in (output_path, Path(".github")): + if path.exists(): + shutil.rmtree(path) + shutil.copytree(source_path, output_path) + print(f"Initialized {output_root} from {source_dir}") + + +def make_zip_archive(source_dir: Path, zip_path: Path) -> None: + """Pack contents of source_dir into zip_path (files at archive root, no tag prefix).""" + source_dir = source_dir.resolve() + if not source_dir.is_dir(): + raise FileNotFoundError(f"Not a directory: {source_dir}") + zip_path.parent.mkdir(parents=True, exist_ok=True) + if zip_path.exists(): + zip_path.unlink() + with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: + for path in sorted(source_dir.rglob("*")): + if path.is_file(): + arcname = path.relative_to(source_dir) + zf.write(path, arcname) + + +def finalize_zip_output(base: Path, zip_path: Path, suffix: str = "") -> None: + """Create zip from staging dir and remove staging dir.""" + make_zip_archive(base, zip_path) + shutil.rmtree(base) + print(f"Created {zip_path}{suffix}", flush=True) + + +def main(): + import argparse + parser = argparse.ArgumentParser(description="Apply YAML patch to Envgene.yml") + parser.add_argument( + "--output-dir", + default="extended_github_instance_pipeline", + help="Parent directory for artifacts. Default: extended_github_instance_pipeline. " + "With --output-format zip (default), writes /.zip; with dir, " + "//. Tag from DOCKER_IMAGE_TAG or INSTANCE_REPO_PIPELINE_IMAGE_TAG.", + ) + parser.add_argument( + "--output-format", + choices=("zip", "dir"), + default="zip", + help="zip: stage under // then pack to /.zip " + "and remove the folder. dir: keep the versioned directory (no zip).", + ) + parser.add_argument( + "--init-from", + default="/opt/github", + help="Before applying patches, remove // and .github, copy this " + "source into the versioned output directory. Default: /opt/github. Use --no-init to skip.", + ) + parser.add_argument( + "--no-init", + action="store_true", + help="Skip init step (do not rm/cp). Use when output-dir already exists.", + ) + parser.add_argument( + "patch", + nargs="*", + help="Patch file(s) (e.g. components/component-a.yaml components/variables.yaml). " + "If omitted, only the base workflow is copied (no patches).", + ) + args = parser.parse_args() + + version_tag = sanitize_tag(resolve_version_tag()) + parent = Path(args.output_dir) + base = parent / version_tag + zip_path = parent / f"{version_tag}.zip" + all_results = [] + + try: + if not args.patch: + if not args.no_init: + if args.output_format == "zip" and zip_path.exists(): + zip_path.unlink() + init_output_dir(base, args.init_from) + if args.output_format == "zip" and base.exists(): + finalize_zip_output( + base, + zip_path, + " (no component patches to apply)", + ) + elif base.exists(): + print( + f"Initialized {base} (no component patches to apply).", + flush=True, + ) + else: + print( + "No work done (--no-init and staging directory missing).", + flush=True, + ) + return + + if not args.no_init: + if args.output_format == "zip" and zip_path.exists(): + zip_path.unlink() + init_output_dir(base, args.init_from) + + for patch_path in args.patch: + patch_path = Path(patch_path) + if not patch_path.is_file(): + raise FileNotFoundError(f"File not found: {patch_path}") + all_results.extend(apply_patch(patch_path, base)) + if all_results: + print("Applied:") + for r in all_results: + print(f" - {r}") + if args.output_format == "zip" and base.exists(): + finalize_zip_output(base, zip_path) + except (FileNotFoundError, ValueError, KeyError) as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/github_workflows/instance-repo-pipeline/extend_logic/scripts/git_commit.py b/github_workflows/instance-repo-pipeline/extend_logic/scripts/git_commit.py new file mode 100755 index 000000000..a3dba4f6d --- /dev/null +++ b/github_workflows/instance-repo-pipeline/extend_logic/scripts/git_commit.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +""" +git_commit.py - Commit and push changes from pipeline (GitLab CI). + +Requires: GITLAB_TOKEN, CI_PROJECT_DIR +Optional: GITLAB_CI_USER (default: oauth2), CI_SERVER_HOST, CI_PROJECT_PATH, CI_COMMIT_REF_NAME + PIPELINE_OUTPUT_DIR (default: extended_github_instance_pipeline) - output folder name for commit message +""" + +import os +import subprocess +import sys + + +def run(cmd, check=True, env=None): + """Run command (list of args), return completed process.""" + env = env or os.environ.copy() + result = subprocess.run(cmd, env=env) + if check and result.returncode != 0: + sys.exit(result.returncode) + return result + + +def main(): + token = os.environ.get("GITLAB_TOKEN") + if not token: + print("Error: GITLAB_TOKEN is not set", file=sys.stderr) + sys.exit(1) + + project_dir = os.environ.get("CI_PROJECT_DIR") + if not project_dir: + print("Error: CI_PROJECT_DIR is not set", file=sys.stderr) + sys.exit(1) + + os.chdir(project_dir) + + ci_user = os.environ.get("GITLAB_CI_USER", "oauth2") + server_host = os.environ.get("CI_SERVER_HOST", "gitlab.com") + project_path = os.environ.get("CI_PROJECT_PATH") + ref_name = os.environ.get("CI_COMMIT_REF_NAME", "main") + + # CI_PROJECT_PATH not set (e.g. running locally) — derive from git remote + if not project_path: + result = subprocess.run( + ["git", "remote", "get-url", "origin"], + capture_output=True, + text=True, + ) + if result.returncode == 0 and result.stdout.strip(): + url = result.stdout.strip() + # Parse https://host/ns/proj.git or git@host:ns/proj.git + if "://" in url: + path = url.split("://", 1)[1].split("/", 1)[-1] + elif ":" in url: + path = url.split(":", 1)[1] + else: + path = "" + path = path.rstrip("/").removesuffix(".git").strip("/") + if path and path != ".git": + project_path = path + if not project_path: + print( + "Error: CI_PROJECT_PATH not set and could not derive from git remote.\n" + " Set CI_PROJECT_PATH (e.g. 'group/project') or fix origin:\n" + " git remote set-url origin https://gitlab.com/OWNER/REPO.git", + file=sys.stderr, + ) + sys.exit(1) + + print(f"Push: token OK, user={ci_user}, pwd={os.getcwd()}", flush=True) + + env = os.environ.copy() + env["HOME"] = "/tmp" + + run(["git", "config", "--global", "--add", "safe.directory", project_dir], env=env) + run(["git", "config", "user.email", "ci@gitlab"], env=env) + run(["git", "config", "user.name", "GitLab CI"], env=env) + + origin_url = f"https://{ci_user}:{token}@{server_host}/{project_path}.git" + run(["git", "remote", "set-url", "origin", origin_url], env=env) + + run(["git", "add", "."], env=env) + + result = subprocess.run( + ["git", "diff", "--staged", "--quiet"], + env=env, + capture_output=True, + ) + + if result.returncode != 0: + output_dir = os.environ.get("PIPELINE_OUTPUT_DIR", "extended_github_instance_pipeline") + run(["git", "commit", "-m", f"chore: update {output_dir} from pipeline [ci skip]"], env=env) + result = subprocess.run( + ["git", "push", "origin", f"HEAD:{ref_name}"], + env=env, + ) + if result.returncode != 0: + print( + "Error: git push failed - check token scope (write_repository)", + file=sys.stderr, + ) + sys.exit(1) + else: + print("No changes to commit") + + +if __name__ == "__main__": + main() diff --git a/gsf_packages/envgene_discovery_project/git-system-follower-package/package.yaml b/gsf_packages/envgene_discovery_project/git-system-follower-package/package.yaml index 1ae54f281..06bb4e8c3 100644 --- a/gsf_packages/envgene_discovery_project/git-system-follower-package/package.yaml +++ b/gsf_packages/envgene_discovery_project/git-system-follower-package/package.yaml @@ -1,5 +1,5 @@ apiVersion: v1 type: gitlab-ci-pipeline name: envgene_discovery_project -version: 1.21.2 +version: 1.32.6 dependencies: [] diff --git a/gsf_packages/envgene_discovery_project/git-system-follower-package/scripts/init.py b/gsf_packages/envgene_discovery_project/git-system-follower-package/scripts/init.py index e0625c116..d014d73f8 100644 --- a/gsf_packages/envgene_discovery_project/git-system-follower-package/scripts/init.py +++ b/gsf_packages/envgene_discovery_project/git-system-follower-package/scripts/init.py @@ -3,7 +3,49 @@ from git_system_follower.develop.api.templates import create_template, get_template_names # Protected files that should never be deleted -PROTECTED_FILES = {'history.log', '.gitlab-ci.yml', '.gitignore'} +PROTECTED_FILES = { + 'history.log', + '.gitlab-ci.yml', + '.gitignore', + 'gitlab-ci/pipeline_vars.yaml', + 'gitlab-ci/pipeline_vars.yml', +} + + +def _migrate_pipeline_vars_format(content: bytes) -> bytes: + """Migrate old .pipeline_vars: wrapper format to new format.""" + text = content.decode('utf-8') + lines = text.splitlines() + + if not lines: + return content + + # Find .pipeline_vars: - can be anywhere in file (after ---, comments, etc.) + start_idx = None + for i, line in enumerate(lines): + if line.strip() == '.pipeline_vars:': + start_idx = i + break + if start_idx is None: + return content # Not old format + + rest = lines[start_idx + 1:] + non_empty = [line for line in rest if line.strip()] + if not non_empty: + return b'---\n' + + min_indent = min(len(line) - len(line.lstrip()) for line in non_empty) + dedented = [] + for line in rest: + if line.strip() and len(line) - len(line.lstrip()) >= min_indent: + dedented.append(line[min_indent:]) + else: + dedented.append(line) + + result = '\n'.join(dedented) + if not result.strip().startswith('---'): + result = '---\n' + result + return result.encode('utf-8') def _delete_files_from_history(parameters: Parameters): @@ -14,15 +56,14 @@ def _delete_files_from_history(parameters: Parameters): history_log_path = cookiecutter_template_dir / 'history.log' if not history_log_path.exists(): - print(f'Warning: history.log not found at {history_log_path}') + print(f'\t\tWarning: history.log not found at {history_log_path}') return - - # Read files from history.log + try: with open(history_log_path, 'r', encoding='utf-8') as f: files_to_delete = {line.strip() for line in f if line.strip()} except Exception as e: - print(f'Warning: Could not read history.log: {e}') + print(f'\t\tWarning: Could not read history.log: {e}') return if not files_to_delete: @@ -47,9 +88,9 @@ def _delete_files_from_history(parameters: Parameters): directories_to_check.add(file_path.parent) file_full_path.unlink() deleted_count += 1 - print(f'Deleted file: {file_path_str}') + print(f'\t\tDeleted file: {file_path_str}') except Exception as e: - print(f'Warning: Could not delete file {file_path_str}: {e}') + print(f'\t\tWarning: Could not delete file {file_path_str}: {e}') # Delete empty directories for dir_path in sorted(directories_to_check, key=lambda p: len(p.parts), reverse=True): @@ -58,12 +99,12 @@ def _delete_files_from_history(parameters: Parameters): try: if not any(dir_full_path.iterdir()): dir_full_path.rmdir() - print(f'Deleted empty directory: {dir_path}') + print(f'\t\tDeleted empty directory: {dir_path}') except Exception: pass if deleted_count > 0: - print(f'Deleted {deleted_count} file(s) from repository') + print(f'\t\tDeleted {deleted_count} file(s) from repository') def main(parameters: Parameters): @@ -84,17 +125,46 @@ def main(parameters: Parameters): variables = parameters.extras.copy() variables.pop('TEMPLATE', None) - create_template(parameters, template, variables) - # Use current working directory as repository root + # Backup pipeline_vars before create_template - never overwrite user's file repo_root = Path.cwd() + pipeline_vars_paths = ['gitlab-ci/pipeline_vars.yaml', 'gitlab-ci/pipeline_vars.yml'] + pipeline_vars_backup = {} + for f in pipeline_vars_paths: + path = repo_root / f + if path.exists() and path.is_file(): + pipeline_vars_backup[f] = path.read_bytes() + print(f'\t\tFile {f} exists. Backed up for preserve') + + create_template(parameters, template, variables) + + # Restore pipeline_vars if it existed - never overwrite, keep only user's format + if pipeline_vars_backup: + for f in pipeline_vars_paths: + if f in pipeline_vars_backup: + path = repo_root / f + content = pipeline_vars_backup[f] + migrated = _migrate_pipeline_vars_format(content) + if migrated != content: + print(f'\t\tFile {f} migrated to new format. Preserved') + else: + print(f'\t\tFile {f} preserved. Skip overwrite') + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(migrated) + # Remove the other format - template creates yaml, user may have yml + for f in pipeline_vars_paths: + if f not in pipeline_vars_backup: + path = repo_root / f + if path.exists(): + path.unlink() + print(f'\t\tRemoved {f} (user uses different format)') + internal_files_to_remove = ['history.log'] - for file_name in internal_files_to_remove: file_path = repo_root / file_name if file_path.exists(): try: file_path.unlink() - print(f'Removed internal package file: {file_name}') + print(f'\t\tRemoved internal file: {file_name}') except Exception as e: - print(f'Warning: Could not remove {file_name}: {e}') + print(f'\t\tWarning: Could not remove {file_name}: {e}') diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/package.yaml b/gsf_packages/envgene_instance_project/git-system-follower-package/package.yaml index df9796cc0..8fe0e23a5 100644 --- a/gsf_packages/envgene_instance_project/git-system-follower-package/package.yaml +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/package.yaml @@ -1,5 +1,5 @@ apiVersion: v1 type: gitlab-ci-pipeline name: envgene_instance_project -version: 1.21.2 +version: 1.32.6 dependencies: [] diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/init.py b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/init.py index e0625c116..bf282dd75 100644 --- a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/init.py +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/init.py @@ -3,7 +3,49 @@ from git_system_follower.develop.api.templates import create_template, get_template_names # Protected files that should never be deleted -PROTECTED_FILES = {'history.log', '.gitlab-ci.yml', '.gitignore'} +PROTECTED_FILES = { + 'history.log', + '.gitlab-ci.yml', + '.gitignore', + 'gitlab-ci/pipeline_vars.yaml', + 'gitlab-ci/pipeline_vars.yml', +} + + +def _migrate_pipeline_vars_format(content: bytes) -> bytes: + """Migrate old .pipeline_vars: wrapper format to new format.""" + text = content.decode('utf-8') + lines = text.splitlines() + + if not lines: + return content + + # Find .pipeline_vars: - can be anywhere in file (after ---, comments, etc.) + start_idx = None + for i, line in enumerate(lines): + if line.strip() == '.pipeline_vars:': + start_idx = i + break + if start_idx is None: + return content # Not old format + + rest = lines[start_idx + 1:] + non_empty = [line for line in rest if line.strip()] + if not non_empty: + return b'---\n' + + min_indent = min(len(line) - len(line.lstrip()) for line in non_empty) + dedented = [] + for line in rest: + if line.strip() and len(line) - len(line.lstrip()) >= min_indent: + dedented.append(line[min_indent:]) + else: + dedented.append(line) + + result = '\n'.join(dedented) + if not result.strip().startswith('---'): + result = '---\n' + result + return result.encode('utf-8') def _delete_files_from_history(parameters: Parameters): @@ -14,15 +56,14 @@ def _delete_files_from_history(parameters: Parameters): history_log_path = cookiecutter_template_dir / 'history.log' if not history_log_path.exists(): - print(f'Warning: history.log not found at {history_log_path}') + print(f'\t\tWarning: history.log not found at {history_log_path}') return - - # Read files from history.log + try: with open(history_log_path, 'r', encoding='utf-8') as f: files_to_delete = {line.strip() for line in f if line.strip()} except Exception as e: - print(f'Warning: Could not read history.log: {e}') + print(f'\t\tWarning: Could not read history.log: {e}') return if not files_to_delete: @@ -47,9 +88,9 @@ def _delete_files_from_history(parameters: Parameters): directories_to_check.add(file_path.parent) file_full_path.unlink() deleted_count += 1 - print(f'Deleted file: {file_path_str}') + print(f'\t\tDeleted file: {file_path_str}') except Exception as e: - print(f'Warning: Could not delete file {file_path_str}: {e}') + print(f'\t\tWarning: Could not delete file {file_path_str}: {e}') # Delete empty directories for dir_path in sorted(directories_to_check, key=lambda p: len(p.parts), reverse=True): @@ -58,12 +99,12 @@ def _delete_files_from_history(parameters: Parameters): try: if not any(dir_full_path.iterdir()): dir_full_path.rmdir() - print(f'Deleted empty directory: {dir_path}') + print(f'\t\tDeleted empty directory: {dir_path}') except Exception: pass if deleted_count > 0: - print(f'Deleted {deleted_count} file(s) from repository') + print(f'\t\tDeleted {deleted_count} file(s) from repository') def main(parameters: Parameters): @@ -84,17 +125,46 @@ def main(parameters: Parameters): variables = parameters.extras.copy() variables.pop('TEMPLATE', None) - create_template(parameters, template, variables) - # Use current working directory as repository root + # Backup pipeline_vars if exists - do not overwrite user's file repo_root = Path.cwd() + pipeline_vars_paths = ['gitlab-ci/pipeline_vars.yaml', 'gitlab-ci/pipeline_vars.yml'] + pipeline_vars_backup = {} + for f in pipeline_vars_paths: + path = repo_root / f + if path.exists() and path.is_file(): + pipeline_vars_backup[f] = path.read_bytes() + print(f'\t\tFile {f} exists. Backed up for preserve') + + create_template(parameters, template, variables) + + # Restore pipeline_vars if it existed - never overwrite, keep only user's format + if pipeline_vars_backup: + for f in pipeline_vars_paths: + if f in pipeline_vars_backup: + path = repo_root / f + content = pipeline_vars_backup[f] + migrated = _migrate_pipeline_vars_format(content) + if migrated != content: + print(f'\t\tFile {f} migrated to new format. Preserved') + else: + print(f'\t\tFile {f} preserved. Skip overwrite') + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(migrated) + # Remove the other format - template creates yaml, user may have yml + for f in pipeline_vars_paths: + if f not in pipeline_vars_backup: + path = repo_root / f + if path.exists(): + path.unlink() + print(f'\t\tRemoved {f} (user uses different format)') + internal_files_to_remove = ['history.log'] - for file_name in internal_files_to_remove: file_path = repo_root / file_name if file_path.exists(): try: file_path.unlink() - print(f'Removed internal package file: {file_name}') + print(f'\t\tRemoved internal file: {file_name}') except Exception as e: - print(f'Warning: Could not remove {file_name}: {e}') + print(f'\t\tWarning: Could not remove {file_name}: {e}') diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/cookiecutter.json b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/cookiecutter.json index 7198d12d4..f7f4a1136 100644 --- a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/cookiecutter.json +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/cookiecutter.json @@ -2,11 +2,15 @@ "gsf_repository_name": "envgene_instance_project", "docker_registry": "ghcr.io", "docker_namespace": "netcracker", - "envgene_version": "1.21.2", + "envgene_version": "1.32.6", "envgen_image": "qubership-envgene", "pipe_image": "qubership-pipegene", "cloud_deploytool_image": "env-generator-deploytool_build_deploytool", "effective_set_generator_image": "qubership-effective-set-generator", - "gitlab_runner_tag_name": "gitlab-org-docker" + "gitlab_runner_tag_name": "gitlab-org-docker", + "self_token": "", + "_copy_without_render": [ + "example" + ] } diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/.gitlab-ci.yml b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/.gitlab-ci.yml index 2a3d0ae1d..05b5aaa00 100644 --- a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/.gitlab-ci.yml +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/.gitlab-ci.yml @@ -32,9 +32,6 @@ default: .common_module.variables: variables: - module_ansible_dir: "/module/ansible" - module_inventory: "${CI_PROJECT_DIR}/configuration/inventory.yaml" - module_ansible_cfg: "/module/ansible/ansible.cfg" module_config_default: "/module/templates/defaults.yaml" .images.variables: diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/configuration/.gitkeep b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/configuration/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/configuration/artifact_definitions/.gitkeep b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/configuration/artifact_definitions/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/configuration/credentials/.gitkeep b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/configuration/credentials/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/environments/.gitkeep b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/environments/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/README.md b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/README.md new file mode 100644 index 000000000..41686cc7d --- /dev/null +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/README.md @@ -0,0 +1,624 @@ +# Getting Started with EnvGene: Your First Environment + +- [Getting Started with EnvGene: Your First Environment](#getting-started-with-envgene-your-first-environment) + - [What You'll Learn](#what-youll-learn) + - [Prerequisites](#prerequisites) + - [Scenario](#scenario) + - [What is a Template?](#what-is-a-template) + - [What is an Instance?](#what-is-an-instance) + - [How They Work Together](#how-they-work-together) + - [Part 1: Creating Your First Template](#part-1-creating-your-first-template) + - [Step 1: Explore the Sample Template](#step-1-explore-the-sample-template) + - [Step 2: Understand Template Structure](#step-2-understand-template-structure) + - [Step 3: Build Your First Template Artifact](#step-3-build-your-first-template-artifact) + - [Understanding the Template Artifact](#understanding-the-template-artifact) + - [Part 2: Creating Your First Environment Instance](#part-2-creating-your-first-environment-instance) + - [Step 4: Set Up Configuration Folder](#step-4-set-up-configuration-folder) + - [Step 5: Set Up environments Folder](#step-5-set-up-environments-folder) + - [Step 6: Review Platform Environment Configuration](#step-6-review-platform-environment-configuration) + - [Step 7: Review Shared Credentials](#step-7-review-shared-credentials) + - [Step 8: Build the Platform Environment](#step-8-build-the-platform-environment) + - [Step 9: Verify the Results](#step-9-verify-the-results) + - [Part 3: Building a Business Environment](#part-3-building-a-business-environment) + - [Step 10: Review Solution Environment Configuration](#step-10-review-solution-environment-configuration) + - [Step 11: Review Environment-Specific Parameters](#step-11-review-environment-specific-parameters) + - [Step 12: Build the Solution Environment](#step-12-build-the-solution-environment) + - [The Complete Flow](#the-complete-flow) + - [The Flow](#the-flow) + - [Summary](#summary) + - [Reference Documentation](#reference-documentation) + +> [!NOTE] +> **You are in an Instance Repository.** This guide covers both template creation (Part 1, done in a Template Repository) and instance configuration (Parts 2-3, done here). + +**Repository Context:** This readme can be used in both Template and Instance repositories. The instructions will guide you based on where you are. + +- **Part 1** - Creating a Template (done in a Template Repository) +- **Part 2–3** - Creating an Instance (done in an Instance Repository) + +By the end, you will have a working environment and understand how the pipeline uses your configuration. + +## What You'll Learn + +- The relationship between Templates and Instances +- How to create and publish a template artifact +- How to configure an environment +- How the pipeline processes your configuration + +## Prerequisites + +- Access to GitLab with CI/CD enabled +- Basic understanding of YAML syntax +- Git command-line knowledge + +> [!NOTE] +> **Choose your path based on where you are:** +> +> - **In a Template Repository?** Follow Part 1 to create and publish templates (requires write access to registry) +> - **In an Instance Repository?** Skip to Part 2 to configure environments (requires read access to registry) + +## Scenario + +You are setting up your first EnvGene environment in `cluster-01`. You will create two environments: + +- **platform-env** - Infrastructure services (ArangoDB, Consul, OpenSearch) +- **solution-env** - Business applications (BSS, OSS, Core) + +By the end, you will have generated the Effective Set and understand how template + instance work together. + +### What is a Template? + +A **Template** is a reusable blueprint that defines solution structure (what namespaces exist) and default configuration parameters. + +### What is an Instance? + +An **Instance** is a specific environment created from a template. It adds environment-specific values and overrides default parameters to produce deployable configuration. + +### How They Work Together + +```mermaid +flowchart LR + A[Template Repository] -->|Builds| B[Template Artifact] + B -->|Referenced by| C[Instance Repository] + C -->|Generates| D[Effective Set] + D -->|Deployed to| E[Kubernetes Cluster] +``` + +1. Template repository builds a versioned artifact +2. Instance repository references that artifact +3. Pipeline merges template + overrides = effective set +4. Effective set is used to deploy to cluster + +## Part 1: Creating Your First Template + +> [!NOTE] +> Part 1 requires access to a Template Repository (a separate repository) with CI/CD enabled. If you already have a published template artifact, skip to Part 2. + +Part 1 is done in a **template repository**, not in this instance repository. Do Part 1 only if you need to create and publish a template artifact. + +### Step 1: Explore the Sample Template + +Navigate to the example folder: + +```bash +cd example/templates/ +ls -la +``` + +You should see: + +```text +templates/ +├── env_templates/ +│ ├── Namespaces/ +│ │ ├── platform/ +│ │ │ ├── arangodb.yml.j2 +│ │ │ ├── consul.yml.j2 +│ │ │ └── opensearch.yml.j2 +│ │ ├── solution/ +│ │ │ ├── bss.yml.j2 +│ │ │ ├── oss.yml.j2 +│ │ │ └── core.yml.j2 +│ │ ├── tenant.yml.j2 +│ │ └── cloud.yml.j2 +│ │ +│ ├── platform.yml +│ └── solution.yml +│ +├── appdefs/ +├── regdefs/ +├── parameters/ +└── resource_profile/ +``` + +### Step 2: Understand Template Structure + +Let's look at a key file, a Template Descriptor - `env_templates/platform.yml`: + +```yaml +tenant: "{{ templates_dir }}/env_templates/Namespaces/tenant.yml.j2" +cloud: "{{ templates_dir }}/env_templates/Namespaces/cloud.yml.j2" +namespaces: + - template_path: "{{ templates_dir }}/env_templates/Namespaces/platform/arangodb.yml.j2" + - template_path: "{{ templates_dir }}/env_templates/Namespaces/platform/consul.yml.j2" + - template_path: "{{ templates_dir }}/env_templates/Namespaces/platform/dbaas.yml.j2" + - template_path: "{{ templates_dir }}/env_templates/Namespaces/platform/jaeger.yml.j2" + - template_path: "{{ templates_dir }}/env_templates/Namespaces/platform/kafka.yml.j2" + - template_path: "{{ templates_dir }}/env_templates/Namespaces/platform/opensearch.yml.j2" +``` + +This file defines the structure of an environment by linking all required component templates. It determines what gets deployed, ensures consistency across environments, and serves as the blueprint referenced by the Environment Inventory. + +Now look at Namespace template `env_templates/Namespaces/platform/consul.yml.j2`: + +```yaml +name: "consul" +credentialsId: "" +isServerSideMerge: false +labels: [] +cleanInstallApprovalRequired: false +mergeDeployParametersAndE2EParameters: true +deployParameters: {} +e2eParameters: {} +technicalConfigurationParameters: {} +deployParameterSets: + - "consul" +e2eParameterSets: [] +technicalConfigurationParameterSets: [] +``` + +- `.j2` extension means it's a Jinja2 template +- It will be rendered with variables you provide (this example omits Jinja syntax for simplicity) + +### Step 3: Build Your First Template Artifact + +Copy the sample template to your templates directory: + +```bash +cp -r example/templates/* templates/ +git add templates/ +git commit -m "feat: Add sample template" +git push +``` + +**Watch the pipeline:** + +1. Go to your GitLab project → CI/CD → Pipelines +2. You should see a pipeline running with these jobs: + - `dp_build` - Building and validating templates + - `report_artifacts` - Recording what was built + - `semantic_release` - Publishing the artifact + +```mermaid +flowchart LR + A[dp_build] --> B[report_artifacts] --> C[semantic_release] +``` + +**Promote / Release Pipeline:** + +Once the `semantic_release` job completes successfully, it automatically triggers a new pipeline responsible for promoting the published template artifact to the release stage. + +```mermaid +flowchart LR + A[prepromote] --> B[promote] + B --> C[update_dependent_versions] + C --> D[get_release_notes] + D --> E[update_release_notes] + E --> F[Release Completed] +``` + +**Jobs:** + +- `prepromote` → Validates promotion conditions. +- `promote` → Promotes to release branch. +- `update_dependent_versions` → Updates dependent modules. +- `get_release_notes` → Extracts commit-based release notes. +- `update_release_notes` → Publishes release documentation. + +Wait for the pipeline to complete (usually 2-5 minutes). + +### Understanding the Template Artifact + +Your pipeline created a template artifact with a version like: + +```text +artifact: template-project:feature-template-sample-20260303.022442-18 +``` + +> [!NOTE] +> Your actual version will differ - it's based on your commit timestamp and build number. + +This artifact now contains all your template files in a packaged format, ready to be used by instances. + +## Part 2: Creating Your First Environment Instance + +From here on, all steps are done **in this repository**. Copy the sample configuration and environments from the `example/` folder in this repository. + +### Step 4: Set Up Configuration Folder + +Copy the configuration folder from `example/`: + +```bash +cp -r example/configuration configuration/ +``` + +The folder structure should look like this: + +``` text +configuration/ +├── artifact_definitions/ +│ └── .yaml +├── credentials/ +│ └── credentials.yml +└── config.yml +``` + +- `.yaml` - Describes where the Environment Template artifact is stored in the registry. It is used to resolve the `application:version` format into the registry and Maven artifact parameters required to download it. + +> [!IMPORTANT] +> +> - All fields inside this file must be replaced with actual project-specific values. + +- `credentials.yml` - Stores credential definitions. +- `config.yml` - Defines EnvGene configuration. + +### Step 5: Set Up environments Folder + +Copy the sample environment: + +```bash +cp -r example/environments/cluster-01 environments/ +``` + +### Step 6: Review Platform Environment Configuration + +The `env_definition.yml` defines which template to use, which artifact version, and what parameters to override. Review its structure: + +Open `environments/cluster-01/platform-env/Inventory/env_definition.yml`: + +```yaml +inventory: {} +envTemplate: + name: "platform" + artifact: "template-project:feature-template-sample-20260303.022442-18" + additionalTemplateVariables: + use_env_prefix: true + sharedTemplateVariables: [] + envSpecificParamsets: {} + envSpecificE2EParamsets: {} + envSpecificTechnicalParamsets: {} + envSpecificResourceProfiles: {} + sharedMasterCredentialFiles: + - "share-creds" +``` + +Comments from the real file are not shown here for simplicity. + +**What each field means:** + +- `inventory`: Defines environment metadata and configuration (tenant, cloud, credentials, etc.). Acts as the recipe for creating the environment. +- `envTemplate.name`: Name of the Environment Template to use. Must match the Template Descriptor name inside the referenced artifact. +- `envTemplate.artifact`: Template artifact in application:version format used to render the environment. +- `additionalTemplateVariables`: Extra variables passed to templates during rendering. +- `sharedTemplateVariables`: Common variable files merged into template variables. +- `envSpecificParamsets`: Environment-specific deployment parameter overrides. +- `envSpecificE2EParamsets`: Environment-specific CI/CD or pipeline parameters. +- `envSpecificTechnicalParamsets`: Runtime/technical parameter overrides. +- `envSpecificResourceProfiles`: Resource (CPU/memory) overrides for this environment. + +> [!IMPORTANT] +> The `envTemplate.artifact` value must match what your template pipeline produced. + +### Step 7: Review Shared Credentials + +Shared credentials are referenced in `env_definition.yml` (via `sharedMasterCredentialFiles`) and used across all environments in this cluster. Let's look at the format: + +Open `environments/cluster-01/credentials/share-creds.yml`: + +```yaml +registry-cred: + type: "usernamePassword" + data: + username: "user-placeholder-123" + password: "pass-placeholder-123" +sso-idp-admin-login: + type: "secret" + data: + secret: "token-placeholder-123" +streaming: + type: "usernamePassword" + data: + username: "user-placeholder-123" + password: "pass-placeholder-123" +postgres-dba-cred: + type: "usernamePassword" + data: + username: "user-placeholder-123" + password: "pass-placeholder-123" +kafka-monitoring-creds: + type: "usernamePassword" + data: + username: "user-placeholder-123" + password: "pass-placeholder-123" +``` + +**Why this matters:** + +- Credentials are shared across environments in the same cluster +- In production scenarios, these would be encrypted +- For learning, we're using plain text + +### Step 8: Build the Platform Environment + +Commit and push your changes: + +```bash +git add environments/ +git commit -m "Add platform environment configuration" +git push +``` + +Trigger the instance pipeline manually: + +1. Go to GitLab → CI/CD → Pipelines → Run Pipeline +2. To generate the Effective Set for the platform environment, EnvGene needs to know which applications (and versions) are deployed. You provide this via the Solution Descriptor (SD). For platform-env, we pass SD inline via pipeline variables (`SD_SOURCE_TYPE: json` and `SD_DATA`). Later (Step 12) you'll do the same for solution-env with BSS/OSS apps. + + Set these variables: + + ```yaml + ENV_NAMES: cluster-01/platform-env + ENV_BUILDER: true + GENERATE_EFFECTIVE_SET: true + SD_SOURCE_TYPE: json + SD_DATA: + ``` + + ```text + '{"version":2.1,"type":"solutionDeploy","deployMode":"composite","applications":[{"version":"","deployPostfix":"arangodb"},{"version":"","deployPostfix":"consul"},{"version":"","deployPostfix":"opensearch"}]}' + ``` + + Replace each `` with your real `application:version` (e.g. `consul:0.11.8`). + + **What's the Solution Descriptor?** + + - Lists specific application versions to deploy + - `deployPostfix` maps to namespace (bss, oss, core) + +3. Click "Run Pipeline" + +**What's happening:** + +```mermaid +flowchart LR + A[generate_pipeline] --> B[run_generated_pipeline] + B --> C[app_reg_def_render] + C --> D[process_sd] + D --> E[env_builder] + E --> F[generate_effective_set] + F --> G[git_commit] +``` + +- `generate_pipeline` - Generates EnvGene pipeline config +- `app_reg_def_render` - Downloads EnvGene Environment template, renders application and registry definitions +- `process_sd` - Processes Solution Descriptor (SD) from pipeline variables +- `env_builder` - Generates Environment Instance - renders template and merges environment-specific overrides +- `generate_effective_set` - Generates Effective Set - final output +- `git_commit` - Commits results back to repository + +### Step 9: Verify the Results + +After the pipeline completes, pull the changes: + +```bash +git pull +``` + +Look at the generated structure: + +```bash +ls -la environments/cluster-01/platform-env/ +``` + +You should see: + +```text +environments/ +└── cluster-01/ + ├── platform-env/ + │ ├── AppDefs/ + │ ├── RegDefs/ + │ ├── Credentials/ + │ │ └── credentials.yml + │ ├── Inventory/ + │ │ ├── env_definition.yml + │ │ └── solution-descriptor/ + │ │ └── sd.yml + │ ├── Namespaces/ + │ ├── Profiles/ + │ ├── effective-set/ + │ │ ├── deployment/ + │ │ ├── topology/ + │ │ ├── pipeline/ + │ │ ├── runtime/ + │ │ └── cleanup/ + │ ├── cloud.yml + │ └── tenant.yml + │ + └── credentials/ + └── share-creds.yml +``` + +The `effective-set/` folder contains five contexts (deployment, topology, pipeline, runtime, cleanup). For example, open `effective-set/deployment///values/deployment-parameters.yaml` to see the final merged parameters that Helm will use. For details on each context, see [Tutorial: Understanding the Effective Set](https://github.com/Netcracker/qubership-envgene/blob/main/docs/tutorials/effective-set.md). + +## Part 3: Building a Business Environment + +Now let's add a second environment with business applications. + +### Step 10: Review Solution Environment Configuration + +The solution-env uses the "solution" template and references parameter sets for BSS and OSS namespaces. Review its `env_definition.yml`: + +Open `environments/cluster-01/solution-env/Inventory/env_definition.yml`: + +```yaml +inventory: {} +envTemplate: + name: "solution" + artifact: "template-project:feature-template-sample-20260303.022442-18" + additionalTemplateVariables: + use_env_prefix: true + sharedTemplateVariables: [] + envSpecificParamsets: + bss: + - "env-specific-bss" + oss: + - "oss-env-specific" + envSpecificE2EParamsets: {} + envSpecificTechnicalParamsets: {} + envSpecificResourceProfiles: {} + sharedMasterCredentialFiles: [] +``` + +Like before, comments are omitted for clarity. + +**What's different from the previous environment:** + +- `envTemplate.name: solution` - Uses the solution template +- `envSpecificParamsets` - Overrides parameters per namespace (e.g. `bss`, `oss`) +- `sharedMasterCredentialFiles: []` - Solution-env in this example does not use shared credentials + +### Step 11: Review Environment-Specific Parameters + +The solution-env `env_definition.yml` references parameter sets `env-specific-bss` and `oss-env-specific`. The `oss-env-specific.yml` file overrides parameters just for the OSS namespace in this environment. + +Open one of these files `environments/cluster-01/solution-env/Inventory/parameters/oss-env-specific.yml`: + +```yaml +name: oss-env-specific +parameters: + LISTENER_NODE_IP: 10.10.10.10 +applications: [] +``` + +**Why this is powerful:** + +- Template defines defaults +- You override only what's different +- Same template, different values per environment + +### Step 12: Build the Solution Environment + +Now that you understand how solution-env uses template + overrides, let's generate its Effective Set. + +In this step, we build the **solution-env** environment. Set these variables: + +```yaml + ENV_NAMES: cluster-01/solution-env + ENV_BUILDER: true + GENERATE_EFFECTIVE_SET: true + SD_SOURCE_TYPE: json + SD_DATA: +``` + +```text +'{"version":2.1,"type":"solutionDeploy","deployMode":"composite","applications":[{"version":"","deployPostfix":"core"},{"version":"","deployPostfix":"bss"},{"version":"","deployPostfix":"oss"}]}' +``` + +Replace each `` with your real `application:version` (e.g. `core:release-9.16.0`). + +Click "Run Pipeline" + +**What You Now Have:** + +After running pipelines for both environments (Steps 8 and 12), your repository structure looks like this: + +```text +environments/ +└── cluster-01/ + ├── platform-env/ + │ ├── AppDefs/ + │ ├── RegDefs/ + │ ├── Credentials/ + │ ├── Inventory/ + │ │ ├── env_definition.yml + │ │ └── solution-descriptor/ + │ │ └── sd.yml + │ ├── Namespaces/ + │ ├── Profiles/ + │ ├── effective-set/ + │ │ ├── deployment/ + │ │ ├── topology/ + │ │ ├── pipeline/ + │ │ ├── runtime/ + │ │ └── cleanup/ + │ ├── cloud.yml + │ └── tenant.yml + │ + ├── solution-env/ + │ ├── AppDefs/ + │ ├── RegDefs/ + │ ├── Credentials/ + │ ├── Inventory/ + │ │ ├── env_definition.yml + │ │ ├── parameters/ + │ │ └── solution-descriptor/ + │ │ └── sd.yml + │ ├── Namespaces/ + │ ├── Profiles/ + │ ├── effective-set/ + │ │ ├── deployment/ + │ │ ├── topology/ + │ │ ├── pipeline/ + │ │ ├── runtime/ + │ │ └── cleanup/ + │ ├── cloud.yml + │ └── tenant.yml + │ + └── credentials/ + └── share-creds.yml +``` + +For the meaning of Effective Set contexts (deployment, topology, pipeline, runtime, cleanup), see [Tutorial: Understanding the Effective Set](https://github.com/Netcracker/qubership-envgene/blob/main/docs/tutorials/effective-set.md). + +## The Complete Flow + +### The Flow + +```mermaid +flowchart TB + A[Template Repository] -->|1. Builds| B[Template Artifact
sample-template-SNAPSHOT] + B -->|2. Referenced by| C[Instance Repository] + C -->|3. Renders with| D[env_definition.yml] + C -->|4. Merges with| E[parameters/] + C -->|5. Applies| F[Solution Descriptor] + D --> G[env_builder] + E --> G + F --> G + G -->|6. Produces| H[Effective Set] + H -->|7. Ready for| I[Deployment] +``` + +**Congratulations!** You've successfully created your first EnvGene environment from scratch. You now understand the core workflow and are ready to build more complex environments. + +## Summary + +By following this tutorial, you: + +- Created a template artifact in a Template Repository (Part 1) or used an existing one +- Configured an Instance Repository with two environments: `platform-env` and `solution-env` (Part 2-3) +- Passed a Solution Descriptor via pipeline variables to map applications to namespaces +- Generated the Effective Set for both environments and reviewed the file structure +- Understand the flow: Template Artifact → env_definition.yml → parameters → SD → Effective Set → Deployment + +You are now ready to add more environments, customize parameters, and explore advanced features like Resource Profile Overrides ([Tutorial: Managing Resource Profiles](https://github.com/Netcracker/qubership-envgene/blob/main/docs/tutorials/resource-profiles.md)) and Effective Set contexts ([Tutorial: Understanding the Effective Set](https://github.com/Netcracker/qubership-envgene/blob/main/docs/tutorials/effective-set.md)). + +## Reference Documentation + +For detailed documentation on EnvGene objects and configuration: + +- [EnvGene Objects Reference](https://github.com/Netcracker/qubership-envgene/blob/main/docs/envgene-objects.md) +- [Environment Configuration](https://github.com/Netcracker/qubership-envgene/blob/main/docs/envgene-configs.md) +- [Pipeline Parameters](https://github.com/Netcracker/qubership-envgene/blob/main/docs/instance-pipeline-parameters.md) +- [How-to Guides](https://github.com/Netcracker/qubership-envgene/tree/main/docs/how-to) +- [All Documentation](https://github.com/Netcracker/qubership-envgene/tree/main/docs) +- [EnvGene Main readme](https://github.com/Netcracker/qubership-envgene/blob/main/README.md) diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/configuration/artifact_definitions/template-project.yaml b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/configuration/artifact_definitions/template-project.yaml new file mode 100644 index 000000000..3b270ccf3 --- /dev/null +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/configuration/artifact_definitions/template-project.yaml @@ -0,0 +1,12 @@ +# shoud be updated -> add here in README.md +# CMDB case +name: "template-project" +groupId: "mvn.group" +artifactId: "template-project" +registry: + name: "ghcr.io" + mavenConfig: + repositoryDomainName: "https://artifactory.qubership.org/" + targetSnapshot: "maven-target-snapshot" + targetStaging: "maven-target-staging" + targetRelease: "maven-target-release" \ No newline at end of file diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/configuration/config.yml b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/configuration/config.yml new file mode 100644 index 000000000..ffe2508b1 --- /dev/null +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/configuration/config.yml @@ -0,0 +1,17 @@ +# See full reference at https://github.com/Netcracker/qubership-envgene/blob/main/docs/envgene-configs.md#configyml +# +# Optional +# Parameter defines the encryption mode. Default - `true` +crypt: false +# Optional +# Defines the encryption technology. Default - `Fernet` +# crypt_backend: enum [Fernet, SOPS] +# Optional +# SBOM retention configuration +# sbom_retention: + # Optional, default value - false + # If `true`, SBOM retention will be enabled + # enabled: false + # Optional, default value - 10 + # Number of latest versions to keep per application when enabled + # keep_versions_per_app: 10 \ No newline at end of file diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/configuration/credentials/credentials.yml b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/configuration/credentials/credentials.yml new file mode 100644 index 000000000..0a0bcfb0e --- /dev/null +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/configuration/credentials/credentials.yml @@ -0,0 +1,8 @@ +# See full reference at https://github.com/Netcracker/qubership-envgene/blob/main/docs/envgene-objects.md#credential +# +# Self-token credential for EnvGene to commit changes to this repository +# Referenced in `/configuration/integration.yml` +self-token-cred: + type: secret + data: + secret: "token-placeholder-123" diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/configuration/integration.yml b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/configuration/integration.yml new file mode 100644 index 000000000..88f5f9020 --- /dev/null +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/configuration/integration.yml @@ -0,0 +1,22 @@ +# See full reference at https://github.com/Netcracker/qubership-envgene/blob/main/docs/envgene-configs.md#integrationyml +# +# Optional +# Configuration for Cloud Passport discovery integration +# cp_discovery: + # Optional + # Parameters for GitLab-based discovery repository + # gitlab: + # Mandatory + # Full project path of the discovery repository + # project: string + # Mandatory + # Branch name of the discovery repository + # branch: master + # Mandatory + # Authentication token for the discovery repository + # token: string + +# Mandatory +# Authentication token for EnvGene to access this instance repository +# Required for EnvGene to commit changes +self_token: "${creds.get('self-token-cred').secret}" diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/cloud-passport/cluster-01-creds.yml b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/cloud-passport/cluster-01-creds.yml new file mode 100644 index 000000000..e4dee4f23 --- /dev/null +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/cloud-passport/cluster-01-creds.yml @@ -0,0 +1,23 @@ +cloud-deploy-token: + type: "secret" + data: + secret: "token-placeholder-123" +consul-admin-cred: + type: "secret" + data: + secret: "token-placeholder-123" +dbaas-cluster-dba-cred: + type: "usernamePassword" + data: + username: "user-placeholder-123" + password: "pass-placeholder-123" +maas-cred: + type: "usernamePassword" + data: + username: "user-placeholder-123" + password: "pass-placeholder-123" +minio-storage-cred: + type: "usernamePassword" + data: + username: "user-placeholder-123" + password: "pass-placeholder-123" \ No newline at end of file diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/cloud-passport/cluster-01.yml b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/cloud-passport/cluster-01.yml new file mode 100644 index 000000000..afd095fda --- /dev/null +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/cloud-passport/cluster-01.yml @@ -0,0 +1,62 @@ +version: 1.5 +cloud: + CLOUD_API_HOST: api.example-cloud.managed.qubership.org + CLOUD_API_PORT: "6443" + CLOUD_DEPLOY_TOKEN: cloud-deploy-token + CLOUD_PUBLIC_HOST: example-cloud.managed.qubership.org + CLOUD_PRIVATE_HOST: example-cloud.managed.qubership.org + CLOUD_DASHBOARD_URL: https://dashboard.example-cloud.managed.qubership.org + CLOUD_PROTOCOL: https + PRODUCTION_MODE: false + GRAYLOG_UI_URL: https://graylog-logging-example-cloud.managed.qubership.org + TRACING_UI_URL: https://jaeger.example-cloud.managed.qubership.org + GRAFANA_UI_URL: https://grafana-platform-monitoring.example-cloud.managed.qubership.org + CMDB_URL: https://cloud-deployer.qubership.org +dbaas: + API_DBAAS_ADDRESS: http://dbaas-aggregator.dbaas:8080 + DBAAS_AGGREGATOR_ADDRESS: https://aggregator-dbaas.example-cloud.managed.qubership.org + DBAAS_CLUSTER_DBA_CREDENTIALS_USERNAME: ${creds.get("dbaas-cluster-dba-cred").username} + DBAAS_CLUSTER_DBA_CREDENTIALS_PASSWORD: ${creds.get("dbaas-cluster-dba-cred").password} +maas: + MAAS_INTERNAL_ADDRESS: http://maas-service.maas:8080 + MAAS_SERVICE_ADDRESS: http://maas-service.maas:8080 + MAAS_CREDENTIALS_USERNAME: ${creds.get("maas-cred").username} + MAAS_CREDENTIALS_PASSWORD: ${creds.get("maas-cred").password} +consul: + CONSUL_URL: http://consul-server.consul:8500 + CONSUL_ENABLED: true + CONSUL_PUBLIC_URL: https://consul.example-cloud.managed.qubership.org + CONSUL_ADMIN_TOKEN: ${creds.get("consul-admin-cred").secret} +zookeeper: + ZOOKEEPER_URL: ${ZOOKEEPER_ADDRESS} + ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 +storage: + STORAGE_SERVER_URL: https://s3.qubership.org + STORAGE_PROVIDER: s3 + STORAGE_REGION: "us-west-2" + STORAGE_USERNAME: ${creds.get("minio-storage-cred").username} + STORAGE_PASSWORD: ${creds.get("minio-storage-cred").password} + CDN_STORAGE_SERVER_URL: ${STORAGE_SERVER_URL} + CDN_STORAGE_PROVIDER: ${STORAGE_PROVIDER} + CDN_STORAGE_REGION: ${STORAGE_REGION} + CDN_STORAGE_USERNAME: ${STORAGE_USERNAME} + CDN_STORAGE_PASSWORD: ${STORAGE_PASSWORD} + DOC_STORAGE_SERVER_URL: ${STORAGE_SERVER_URL} + DOC_STORAGE_PROVIDER: ${STORAGE_PROVIDER} + DOC_STORAGE_REGION: ${STORAGE_REGION} + DOC_STORAGE_USERNAME: ${STORAGE_USERNAME} + DOC_STORAGE_PASSWORD: ${STORAGE_PASSWORD} + STORAGE_RWO_CLASS: rwo-sc + STORAGE_RWX_CLASS: rwx-sc +core: + DEFAULT_TENANT_NAME: default + DEFAULT_TENANT_ADMIN_LOGIN: user-placeholder-123 + DEFAULT_TENANT_ADMIN_PASSWORD: pass-placeholder-123 + MAVEN_REPO_URL: https://artifactory.qubership.org/ + MAVEN_REPO_NAME: global.mvn.group + MAVEN_REPO_STAGING_NAME: ${MAVEN_REPO_NAME} + MAVEN_REPO_DEV_NAME: ${MAVEN_REPO_NAME} +global: + MONITORING_ENABLED: "true" + TRACING_ENABLED: "true" + TRACING_HOST: diagnostic-agent \ No newline at end of file diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/credentials/share-creds.yml b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/credentials/share-creds.yml new file mode 100644 index 000000000..73381a3d3 --- /dev/null +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/credentials/share-creds.yml @@ -0,0 +1,24 @@ +registry-cred: + type: "usernamePassword" + data: + username: "user-placeholder-123" + password: "pass-placeholder-123" +sso-idp-admin-login: + type: "secret" + data: + secret: "token-placeholder-123" +streaming: + type: "usernamePassword" + data: + username: "user-placeholder-123" + password: "pass-placeholder-123" +postgres-dba-cred: + type: "usernamePassword" + data: + username: "user-placeholder-123" + password: "pass-placeholder-123" +kafka-monitoring-creds: + type: "usernamePassword" + data: + username: "user-placeholder-123" + password: "pass-placeholder-123" \ No newline at end of file diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/platform-env/Inventory/env_definition.yml b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/platform-env/Inventory/env_definition.yml new file mode 100644 index 000000000..de8f5ab14 --- /dev/null +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/platform-env/Inventory/env_definition.yml @@ -0,0 +1,115 @@ +# See full reference at https://github.com/Netcracker/qubership-envgene/blob/main/docs/envgene-configs.md#integrationyml +# +# Mandatory +inventory: {} + # Optional + # Name of the Environment + # environmentName: string + # Optional + # Name of the Tenant for the Environment + # tenantName: string + # Optional + # Name of the Cloud for the Environment + # cloudName: string + # Optional + # URL of the Cluster as specified in kubeconfig + # clusterUrl: string + # Optional + # Reference to Cloud Passport + # cloudPassport: string + # Optional + # Reference to external CMDB system where the Environment Instance can be imported + # deployer: string + # Optional + # Environment description + # description: string + # Optional + # Environment owners + # owners: string + # Optional + # config: + # Optional. Default value - `false` + # If `true`, during CMDB import credentials IDs will be unique for each environment + # updateCredIdsWithEnvName: boolean + # Optional. Default value - `false` + # If `true`, during CMDB import resource profile override names will be unique for each environment + # updateRPOverrideNameWithEnvName: boolean + # Optional. Default value - `true` + # If `true`, environment-specific Resource Profile Overrides defined in envTemplate.envSpecificParamsets + # are merged with Resource Profile Overrides from the Environment Template + # If `false`, they completely replace the Environment Template's Resource Profile Overrides + # mergeEnvSpecificResourceProfiles: boolean +envTemplate: + # Mandatory + # Name of the template in the Environment Template artifact + name: platform + # Mandatory + # Template artifact in application:version notation + artifact: "template-project:feature-template-sample-SNAPSHOT" + # Optional + # bgNsArtifacts: + # Mandatory + # Template artifact in application:version notation for origin namespace + # origin: string + # Mandatory + # Template artifact in application:version notation for peer namespace + # peer: string + # Optional + # Additional variables that will be available during template rendering + # Value is a key-value hashmap + additionalTemplateVariables: + use_env_prefix: true + # Optional + # Array of filenames containing parameters that will be merged with `additionalTemplateVariables` + # Files must contain key-value hashmap + # Example: + # sharedTemplateVariables: + # - prod-template-variables + # - sample-cloud-template-variables + sharedTemplateVariables: [] + # Optional + # Set of environment-specific deployment parameters + # Keys can be either the `cloud` name or the Namespace identifier (which is defined by the `deploy_postfix` + # in the Template Descriptor, or by the Namespace template filename without extension) + # Example: + # envSpecificParamsets: + # bss: + # - env-specific-bss + # - prod-shared + envSpecificParamsets: {} + # Optional + # Environment specific pipeline (e2e) parameters set + # Keys can be either the `cloud` name or the Namespace identifier (which is defined by the `deploy_postfix` + # in the Template Descriptor, or by the Namespace template filename without extension) + # Example: + # envSpecificE2EParamsets: + # cloud: + # - cloud-level-params + envSpecificE2EParamsets: {} + # Optional + # Environment specific technical parameters set + # Keys can be either the `cloud` name or the Namespace identifier (which is defined by the `deploy_postfix` + # in the Template Descriptor, or by the Namespace template filename without extension) + # Example: + # envSpecificTechnicalParamsets: + # bss: + # - env-specific-tech + envSpecificTechnicalParamsets: {} + # Optional + # Environment specific resource profile overrides + # Keys can be either the `cloud` name or the Namespace identifier (which is defined by the `deploy_postfix` + # in the Template Descriptor, or by the Namespace template filename without extension) + # Example: + # envSpecificResourceProfiles: + # bss: + # - bss-override + envSpecificResourceProfiles: {} + # Optional + # Array of filenames containing credentials that will be merged with `additionalTemplateVariables` + # File must contain key-value hashmap + # Example: + # sharedMasterCredentialFiles: + # - prod-integration-creds + # - share-creds + sharedMasterCredentialFiles: + - "share-creds" diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/solution-env/Inventory/env_definition.yml b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/solution-env/Inventory/env_definition.yml new file mode 100644 index 000000000..8fc372cfd --- /dev/null +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/solution-env/Inventory/env_definition.yml @@ -0,0 +1,119 @@ +# See full reference at https://github.com/Netcracker/qubership-envgene/blob/main/docs/envgene-configs.md#integrationyml +# +# Mandatory +inventory: {} + # Optional + # Name of the Environment + # environmentName: string + # Optional + # Name of the Tenant for the Environment + # tenantName: string + # Optional + # Name of the Cloud for the Environment + # cloudName: string + # Optional + # URL of the Cluster as specified in kubeconfig + # clusterUrl: string + # Optional + # Reference to Cloud Passport + # cloudPassport: string + # Optional + # Reference to external CMDB system where the Environment Instance can be imported + # deployer: string + # Optional + # Environment description + # description: string + # Optional + # Environment owners + # owners: string + # Optional + # config: + # Optional. Default value - `false` + # If `true`, during CMDB import credentials IDs will be unique for each environment + # updateCredIdsWithEnvName: boolean + # Optional. Default value - `false` + # If `true`, during CMDB import resource profile override names will be unique for each environment + # updateRPOverrideNameWithEnvName: boolean + # Optional. Default value - `true` + # If `true`, environment-specific Resource Profile Overrides defined in envTemplate.envSpecificParamsets + # are merged with Resource Profile Overrides from the Environment Template + # If `false`, they completely replace the Environment Template's Resource Profile Overrides + # mergeEnvSpecificResourceProfiles: boolean +envTemplate: + # Mandatory + # Name of the template in the Environment Template artifact + name: solution + # Mandatory + # Template artifact in application:version notation + artifact: "template-project:feature-template-sample-SNAPSHOT" + # Optional + # bgNsArtifacts: + # Mandatory + # Template artifact in application:version notation for origin namespace + # origin: string + # Mandatory + # Template artifact in application:version notation for peer namespace + # peer: string + # Optional + # Additional variables that will be available during template rendering + # Value is a key-value hashmap + additionalTemplateVariables: + use_env_prefix: true + # Optional + # Array of filenames containing parameters that will be merged with `additionalTemplateVariables` + # Files must contain key-value hashmap + # Example: + # sharedTemplateVariables: + # - prod-template-variables + # - sample-cloud-template-variables + sharedTemplateVariables: [] + # Optional + # Set of environment-specific deployment parameters + # Keys can be either the `cloud` name or the Namespace identifier (which is defined by the `deploy_postfix` + # in the Template Descriptor, or by the Namespace template filename without extension) + # Example: + # envSpecificParamsets: + # bss: + # - env-specific-bss + # - prod-shared + envSpecificParamsets: + bss: + - "env-specific-bss" + oss: + - "oss-env-specific" + # Optional + # Environment specific pipeline (e2e) parameters set + # Keys can be either the `cloud` name or the Namespace identifier (which is defined by the `deploy_postfix` + # in the Template Descriptor, or by the Namespace template filename without extension) + # Example: + # envSpecificE2EParamsets: + # cloud: + # - cloud-level-params + envSpecificE2EParamsets: {} + # Optional + # Environment specific technical parameters set + # Keys can be either the `cloud` name or the Namespace identifier (which is defined by the `deploy_postfix` + # in the Template Descriptor, or by the Namespace template filename without extension) + # Example: + # envSpecificTechnicalParamsets: + # bss: + # - env-specific-tech + envSpecificTechnicalParamsets: {} + # Optional + # Environment specific resource profile overrides + # Keys can be either the `cloud` name or the Namespace identifier (which is defined by the `deploy_postfix` + # in the Template Descriptor, or by the Namespace template filename without extension) + # Example: + # envSpecificResourceProfiles: + # bss: + # - bss-override + envSpecificResourceProfiles: {} + # Optional + # Array of filenames containing credentials that will be merged with `additionalTemplateVariables` + # File must contain key-value hashmap + # Example: + # sharedMasterCredentialFiles: + # - prod-integration-creds + # - share-creds + sharedMasterCredentialFiles: + - "share-creds" diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/solution-env/Inventory/parameters/cloud-env-specific.yml b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/solution-env/Inventory/parameters/cloud-env-specific.yml new file mode 100644 index 000000000..effd1b81b --- /dev/null +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/solution-env/Inventory/parameters/cloud-env-specific.yml @@ -0,0 +1,6 @@ +name: cloud-env-specific +parameters: + PAAS_PLATFORM: KUBERNETES + PAAS_VERSION: v1.32 + INGRESS_CLASS: nginx +applications: [] \ No newline at end of file diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/solution-env/Inventory/parameters/oss-env-specific.yml b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/solution-env/Inventory/parameters/oss-env-specific.yml new file mode 100644 index 000000000..d72df7679 --- /dev/null +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/example/environments/cluster-01/solution-env/Inventory/parameters/oss-env-specific.yml @@ -0,0 +1,4 @@ +name: oss-env-specific +parameters: + LISTENER_NODE_IP: 10.10.10.10 +applications: [] \ No newline at end of file diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/git_hooks/config.schema.json b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/git_hooks/config.schema.json index c8f27ab93..371d097fa 100644 --- a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/git_hooks/config.schema.json +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/git_hooks/config.schema.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "title": "Generic Configuration", - "description": "Configuration for the for cryptographic operations, artifact discovery, and cloud passport decryption", + "description": "Configuration for cryptographic operations, artifact discovery, SBOM retention, and cloud passport decryption", "additionalProperties": true, "properties": { "crypt": { @@ -57,6 +57,36 @@ "cmdb", "local" ] + }, + "sbom_retention": { + "type": "object", + "title": "SBOM Retention Configuration", + "description": "Configuration for automatic SBOM file cleanup to manage repository size. Triggers only when repository reaches 1200 GB threshold.", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "title": "Enable SBOM Retention", + "description": "Enable or disable SBOM retention cleanup", + "default": false, + "examples": [ + true, + false + ] + }, + "keep_versions_per_app": { + "type": "integer", + "title": "Versions to Keep", + "description": "Number of latest versions to keep per application. Used only when enabled is true.", + "minimum": 1, + "default": 10, + "examples": [ + 5, + 10, + 15 + ] + } + } } } } diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/gitlab-ci/pipeline_vars.yaml b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/gitlab-ci/pipeline_vars.yaml index cc536ab14..86b02682c 100644 --- a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/gitlab-ci/pipeline_vars.yaml +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/gitlab-ci/pipeline_vars.yaml @@ -35,4 +35,4 @@ variables: description: "Will EnvGene generate Effective set" options: - "true" - - "false" + - "false" \ No newline at end of file diff --git a/pom.xml b/pom.xml deleted file mode 100644 index 194abd016..000000000 --- a/pom.xml +++ /dev/null @@ -1,122 +0,0 @@ - - - 4.0.0 - - org.qubership - qubership_envgene_templates - ${revision} - - ${project.groupId}:${project.artifactId} - >A Maven artifact that contains templates for Qubership Envgene Instance - https://github.com/Netcracker/qubership-envgene - - - Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0 - - - - - Netcracker - opensourcegroup@netcracker.com - Netcracker Technology - https://www.netcracker.com - - - - scm:git:git://github.com/Netcracker/qubership-envgene.git - scm:git:ssh://github.com:Netcracker/qubership-envgene.git - https://github.com/Netcracker/qubership-envgene/tree/main - - - - - central - Central Maven Repository - - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 3.2.7 - - - sign-artifacts - verify - - sign - - - - - - - org.apache.maven.plugins - maven-source-plugin - 3.3.0 - - - attach-sources - - jar-no-fork - - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 3.6.3 - - - attach-javadocs - - jar - - - - - - - org.sonatype.central - central-publishing-maven-plugin - 0.6.0 - true - - central - true - published - - - - - org.apache.maven.plugins - maven-assembly-plugin - 3.6.0 - - - make-assembly - package - - single - - - - - - src/assembly/assembly.xml - - false - - - - - - - 0.0.0.1 - - diff --git a/python/artifact-searcher/artifact_searcher/artifact.py b/python/artifact-searcher/artifact_searcher/artifact.py index f237d70e6..934570360 100644 --- a/python/artifact-searcher/artifact_searcher/artifact.py +++ b/python/artifact-searcher/artifact_searcher/artifact.py @@ -1,4 +1,5 @@ import asyncio +import base64 import os import re import shutil @@ -8,14 +9,13 @@ from typing import Any, Optional from urllib.parse import urljoin, urlparse, urlunparse from zipfile import ZipFile +from collections import defaultdict import aiohttp import requests -from aiohttp import BasicAuth from artifact_searcher.utils.constants import DEFAULT_REQUEST_TIMEOUT, TCP_CONNECTION_LIMIT, METADATA_XML -from artifact_searcher.utils.models import Registry, Application, FileExtension, Credentials, ArtifactInfo +from artifact_searcher.utils.models import Registry, RegistryV2, Application, FileExtension, Credentials, ArtifactInfo, Provider, MavenConfig from envgenehelper import logger -from requests.auth import HTTPBasicAuth WORKSPACE = os.getenv("WORKSPACE", Path(tempfile.gettempdir()) / "zips") @@ -35,11 +35,18 @@ def convert_nexus_repo_url_to_index_view(url: str) -> str: return urlunparse(parsed._replace(path=new_path)) -def create_artifact_path(app: Application, version: str, repo: str) -> str: - registry_url = app.registry.maven_config.repository_domain_name.rstrip("/") + "/" +def create_artifact_path(app: Application, version: str, repo: str = "") -> str: + registry_url = app.registry.maven_config.repository_domain_name group_id = app.group_id.replace(".", "/") folder = version_to_folder_name(version) - return urljoin(registry_url, f"{repo}/{group_id}/{app.artifact_id}/{folder}/") + + # For cloud providers (AWS/GCP), repo is empty since repositoryDomainName already contains full path + if repo: + path = f"{repo}/{group_id}/{app.artifact_id}/{folder}/" + else: + path = f"{group_id}/{app.artifact_id}/{folder}/" + + return urljoin(registry_url, path) def create_full_url(app: Application, version: str, repo: str, artifact_extension: FileExtension, @@ -134,27 +141,43 @@ def clean_temp_dir(): os.makedirs(WORKSPACE, exist_ok=True) -async def download_all_async(artifacts_info: list[ArtifactInfo], cred: Credentials | None = None): - auth = BasicAuth(login=cred.username, password=cred.password) if cred else None - connector = aiohttp.TCPConnector(limit=TCP_CONNECTION_LIMIT) - timeout = aiohttp.ClientTimeout(total=DEFAULT_REQUEST_TIMEOUT) - async with aiohttp.ClientSession(connector=connector, timeout=timeout, auth=auth) as session: - async with asyncio.TaskGroup() as tg: - tasks = [tg.create_task(download_async(session, artifact_info)) for artifact_info in artifacts_info] - results = [] - errors = [] - - for i, task in enumerate(tasks): - result = task.result() - if not result or result.local_path is None: - errors.append(f"Task {i}: artifact was not downloaded") - else: - results.append(result) - - if errors: - raise ValueError("Some tasks failed:\n" + "\n".join(errors)) - - return results +def credentials_to_headers(cred: Credentials) -> dict: + # Convert Credentials object to Authorization headers dict. + token = base64.b64encode(f"{cred.username}:{cred.password}".encode()).decode() + return {"Authorization": f"Basic {token}"} + + +async def download_all_async(artifacts_info: list[ArtifactInfo]): + auth_groups = defaultdict(list) + for artifact in artifacts_info: + # Use sorted tuple of auth items as key, or "none" for None + auth_key = tuple(sorted(artifact.auth_headers.items())) if artifact.auth_headers else "none" + auth_groups[auth_key].append(artifact) + + all_results = [] + for auth_key, artifacts in auth_groups.items(): + headers = artifacts[0].auth_headers + connector = aiohttp.TCPConnector(limit=TCP_CONNECTION_LIMIT) + timeout = aiohttp.ClientTimeout(total=DEFAULT_REQUEST_TIMEOUT) + async with aiohttp.ClientSession(connector=connector, timeout=timeout, headers=headers) as session: + async with asyncio.TaskGroup() as tg: + tasks = [tg.create_task(download_async(session, artifact)) for artifact in artifacts] + results = [] + errors = [] + + for i, task in enumerate(tasks): + result = task.result() + if not result or result.local_path is None: + errors.append(f"Task {i}: artifact was not downloaded") + else: + results.append(result) + + if errors: + raise ValueError("Some tasks failed:\n" + "\n".join(errors)) + + all_results.extend(results) + + return all_results def create_app_artifacts_local_path(app_name, app_version): @@ -198,7 +221,8 @@ async def check_artifact_by_full_url_async( classifier: str = "" ) -> tuple[str, tuple[str, str]] | None: repo_value, repo_pointer = repo - if not repo_value: + # Allow empty repo_value for cloud providers (repositoryName), but not for Nexus/Artifactory + if not repo_value and repo_pointer != "repositoryName": logger.warning(f"[Task {task_id}] [Registry: {app.registry.name}] - {repo_pointer} is not configured") return None @@ -230,19 +254,32 @@ async def check_artifact_by_full_url_async( f"[Task {task_id}] [Application: {app.name}: {version}] - Error checking artifact URL {full_url}: {e}") -def get_repo_value_pointer_dict(registry: Registry): - """Permanent set of repositories for searching of artifacts""" +def _is_cloud_provider(registry) -> bool: + """Check if registry uses AWS or GCP provider.""" + if not getattr(registry, "auth_config", None): + return False + for auth_cfg in registry.auth_config.values(): + if getattr(auth_cfg, "provider", None) in (Provider.AWS, Provider.GCP): + return True + return False + + +# TODO: delete after models are refactored to use polymorphism +def get_repo_value_pointer_dict(registry): + """V2 cloud providers: use empty repo (domain already contains full path). + V1/Nexus/Artifactory: use target fields.""" + if _is_cloud_provider(registry): + return {"": "repositoryName"} maven = registry.maven_config - repos = { + return { maven.target_snapshot: "targetSnapshot", maven.target_staging: "targetStaging", maven.target_release: "targetRelease", maven.snapshot_group: "snapshotGroup", } - return repos -def get_repo_pointer(repo_value: str, registry: Registry): +def get_repo_pointer(repo_value: str, registry): repos_dict = get_repo_value_pointer_dict(registry) return repos_dict.get(repo_value) @@ -252,18 +289,19 @@ async def _attempt_check( version: str, artifact_extension: FileExtension, registry_url: str | None = None, - cred: Credentials | None = None, + auth_headers: dict | None = None, classifier: str = "" ) -> Optional[tuple[str, tuple[str, str]]]: repos_dict = get_repo_value_pointer_dict(app.registry) if registry_url: app.registry.maven_config.repository_domain_name = registry_url - auth = BasicAuth(login=cred.username, password=cred.password) if cred else None + session_headers = auth_headers + timeout = aiohttp.ClientTimeout(total=DEFAULT_REQUEST_TIMEOUT) stop_snapshot_event_for_others = asyncio.Event() stop_artifact_event = asyncio.Event() - async with aiohttp.ClientSession(timeout=timeout, auth=auth) as session: + async with aiohttp.ClientSession(timeout=timeout, headers=session_headers) as session: async with asyncio.TaskGroup() as tg: tasks = [ tg.create_task( @@ -288,8 +326,36 @@ async def _attempt_check( return result +def _should_retry_nexus_url(registry) -> bool: + """Check whether the registry needs Nexus-style URL retry.""" + if isinstance(registry, RegistryV2): + return any(cfg.provider == Provider.NEXUS for cfg in registry.auth_config.values()) + return MavenConfig.is_nexus(registry.maven_config.repository_domain_name) + + +async def _retry_with_nexus_url( + app: Application, + version: str, + artifact_extension: FileExtension, + auth_headers: dict | None, + classifier: str = "" +) -> Optional[tuple[str, tuple[str, str]]]: + """Retry artifact check with Nexus index-view URL conversion.""" + original_domain = app.registry.maven_config.repository_domain_name + fixed_domain = convert_nexus_repo_url_to_index_view(original_domain) + if fixed_domain != original_domain: + logger.info(f"Retrying artifact check with edited domain: {fixed_domain}") + result = await _attempt_check(app, version, artifact_extension, fixed_domain, auth_headers, classifier) + if result is not None: + return result + else: + logger.debug("Domain is same after editing, skipping retry") + return None + + async def check_artifact_async( - app: Application, artifact_extension: FileExtension, version: str, cred: Credentials | None = None, + app: Application, artifact_extension: FileExtension, version: str, + auth_headers: dict | None = None, classifier: str = "") -> Optional[tuple[str, tuple[str, str]]] | None: """ Resolves the full artifact URL and the first repository where it was found. @@ -301,26 +367,34 @@ async def check_artifact_async( - tuple[str, str]: A pair of (repository name, repository pointer/alias in CMDB). Returns None if the artifact could not be resolved """ + repos_dict = get_repo_value_pointer_dict(app.registry) - result = await _attempt_check(app, version, artifact_extension, None, cred) - if result is not None: - return result + # Single repo: no parallelism + if len(repos_dict) == 1: + repo_value, repo_pointer = next(iter(repos_dict.items())) + if not repo_value and repo_pointer != "repositoryName": + logger.warning(f"[Registry: {app.registry.name}] - {repo_pointer} is not configured") + return None + domain = app.registry.maven_config.repository_domain_name + repo_url = domain if not repo_value else domain.rstrip('/') + '/' + repo_value + url = check_artifact(repo_url, app.group_id, app.artifact_id, version, + artifact_extension, auth_headers=auth_headers, classifier=classifier) + if url: + return url, (repo_value, repo_pointer) + if _should_retry_nexus_url(app.registry): + return await _retry_with_nexus_url(app, version, artifact_extension, auth_headers, classifier) + return None - if not app.registry.maven_config.is_nexus: + # Multiple repos: use async parallel checking + result = await _attempt_check(app, version, artifact_extension, auth_headers=auth_headers, classifier=classifier) + if result is not None: return result - # trying to edit url for nexus and repeat - original_domain = app.registry.maven_config.repository_domain_name - fixed_domain = convert_nexus_repo_url_to_index_view(original_domain) - if fixed_domain != original_domain: - logger.info(f"Retrying artifact check with edited domain: {fixed_domain}") - result = await _attempt_check(app, version, artifact_extension, fixed_domain, cred, classifier) - if result is not None: - return result - else: - logger.debug("Domain is same after editing, skipping retry") + if _should_retry_nexus_url(app.registry): + return await _retry_with_nexus_url(app, version, artifact_extension, auth_headers, classifier) logger.warning("Artifact not found") + return None def unzip_file(artifact_id: str, app_name: str, app_version: str, zip_url: str): @@ -357,16 +431,19 @@ def create_aql_artifact(app: Application, artifact_extension: FileExtension, ver return aql -def check_artifacts_by_aql(aql: str, cred: Credentials, url: str) -> list[ArtifactInfo]: +def check_artifacts_by_aql(aql: str, url: str = "", + auth_headers: dict | None = None) -> list[ArtifactInfo]: artifacts = [] - response = requests.post(f"{url}/api/search/aql", data=aql, auth=HTTPBasicAuth(cred.username, cred.password)) + base_url = url.rstrip('/') + response = requests.post(f"{base_url}/api/search/aql", data=aql, headers=auth_headers) + response.raise_for_status() results = response.json() - for result in results.get("results"): + for result in (results.get("results") or []): repo = result.get("repo") path = result.get("path") name = result.get("name") - url = f"{url}/{repo}/{path}/{name}" - artifact = ArtifactInfo(repo=repo, path=path, name=name, url=url) + artifact_url = f"{base_url}/{repo}/{path}/{name}" + artifact = ArtifactInfo(repo=repo, path=path, name=name, url=artifact_url, auth_headers=auth_headers) artifacts.append(artifact) return artifacts @@ -375,23 +452,19 @@ def check_artifacts_by_aql(aql: str, cred: Credentials, url: str) -> list[Artifa # TODO delete after deletion feature getting artifact by not artifact def # -------------------------------------------------------------------------------------- -def download_json_content(url: str, cred: Credentials | None = None) -> dict[str, Any]: - auth = HTTPBasicAuth(cred.username, cred.password) if cred else None - response = requests.get( - url, - auth=auth, - timeout=DEFAULT_REQUEST_TIMEOUT - ) +def download_json_content(url: str, auth_headers: dict | None = None) -> dict[str, Any]: + headers = auth_headers + response = requests.get(url, headers=headers, timeout=DEFAULT_REQUEST_TIMEOUT) response.raise_for_status() json_data = response.json() logger.info(f"Got json data by url {url}") return json_data -def download(url: str, target_path: str, cred: Credentials | None = None) -> str: - auth = HTTPBasicAuth(cred.username, cred.password) if cred else None +def download(url: str, target_path: str, auth_headers: dict | None = None) -> str: + headers = auth_headers + response = requests.get(url, headers=headers, timeout=DEFAULT_REQUEST_TIMEOUT) os.makedirs(os.path.dirname(target_path), exist_ok=True) - response = requests.get(url, auth=auth, timeout=DEFAULT_REQUEST_TIMEOUT) response.raise_for_status() with open(target_path, "wb") as f: f.write(response.content) @@ -401,14 +474,17 @@ def download(url: str, target_path: str, cred: Credentials | None = None) -> str def check_artifact(repo_url: str, group_id: str, artifact_id: str, version: str, artifact_extension: FileExtension, - cred: Credentials | None = None, + auth_headers: dict | None = None, classifier: str = "") -> str | None: + if MavenConfig.is_nexus(repo_url): + repo_url = convert_nexus_repo_url_to_index_view(repo_url) + base = repo_url.rstrip("/") + "/" group_id = group_id.replace(".", "/") if "SNAPSHOT" in version: base_path = urljoin(base, f"{group_id}/{artifact_id}/{version}/") - resolved_version = resolve_snapshot_version(base_path, artifact_extension, cred, classifier) + resolved_version = resolve_snapshot_version(base_path, artifact_extension, auth_headers, classifier) if not resolved_version: return None version = resolved_version @@ -416,9 +492,8 @@ def check_artifact(repo_url: str, group_id: str, artifact_id: str, version: str, folder = version_to_folder_name(version) filename = create_artifact_name(artifact_id, artifact_extension, version, classifier) full_url = urljoin(base, f"{group_id}/{artifact_id}/{folder}/{filename}") - try: - response = requests.head(full_url, timeout=DEFAULT_REQUEST_TIMEOUT) + response = requests.head(full_url, headers=auth_headers, timeout=DEFAULT_REQUEST_TIMEOUT) if response.status_code == 200: logger.info( f"[Repository: {repo_url}] [Artifact: {group_id}:{artifact_id}:{version}] - Artifact found: {full_url}" @@ -435,17 +510,13 @@ def check_artifact(repo_url: str, group_id: str, artifact_id: str, version: str, return None -def resolve_snapshot_version(base_path, extension: FileExtension, cred: Credentials | None = None, +def resolve_snapshot_version(base_path, extension: FileExtension, + auth_headers: dict | None = None, classifier: str = "") -> Optional[str]: metadata_url = urljoin(base_path, METADATA_XML) - auth = HTTPBasicAuth(cred.username, cred.password) if cred else None try: - response = requests.get( - metadata_url, - auth=auth, - timeout=DEFAULT_REQUEST_TIMEOUT, - ) + response = requests.get(metadata_url, headers=auth_headers, timeout=DEFAULT_REQUEST_TIMEOUT) if response.status_code != 200: logger.warning(f"Failed to fetch {metadata_url}, status={response.status_code}") return None diff --git a/python/artifact-searcher/artifact_searcher/auth_resolver.py b/python/artifact-searcher/artifact_searcher/auth_resolver.py new file mode 100644 index 000000000..1d0d33738 --- /dev/null +++ b/python/artifact-searcher/artifact_searcher/auth_resolver.py @@ -0,0 +1,139 @@ +import base64 +import json +from typing import Optional + +from artifact_searcher.utils.models import AuthConfig, Provider, RegistryV2 +from envgenehelper import logger + +AUTH_METHOD_USER_PASS = "user_pass" +AUTH_METHOD_SECRET = "secret" +AUTH_METHOD_SERVICE_ACCOUNT = "service_account" +AUTH_METHOD_ANONYMOUS = "anonymous" +AUTH_METHOD_ASSUME_ROLE = "assume_role" +AUTH_METHOD_FEDERATION = "federation" +AUTH_METHOD_OAUTH2 = "oauth2" + +AWS_SERVICE_CODEARTIFACT = "codeartifact" +AWS_TOKEN_KEY = "authorizationToken" +GCP_TOKEN_ATTR = "gcp_authorization_token" + +CRED_FIELD_USERNAME = "username" +CRED_FIELD_PASSWORD = "password" +CRED_FIELD_SECRET = "secret" +CRED_FIELD_DATA = "data" + + +def _get_cred_data(cred_id: str, env_creds: dict) -> dict: + if not env_creds or cred_id not in env_creds: + raise ValueError(f"Credential '{cred_id}' not found in decrypted credentials") + return env_creds[cred_id].get(CRED_FIELD_DATA, {}) + + +def _validate_user_pass_creds(cred_data: dict, context: str) -> tuple[str, str]: + username = cred_data.get(CRED_FIELD_USERNAME) + password = cred_data.get(CRED_FIELD_PASSWORD) + if not username or not password: + raise ValueError(f"{context} requires both username and password in credentials") + return username, password + + +def _aws_bearer(auth_cfg: AuthConfig, cred_data: dict) -> dict: + username, password = _validate_user_pass_creds(cred_data, "AWS secret auth") + + from qubership_pipelines_common_library.v1.utils.utils_aws import AWSCodeArtifactHelper + + token = AWSCodeArtifactHelper.get_authorization_token( + access_key=username, + secret_key=password, + domain=auth_cfg.aws_domain, + region_name=auth_cfg.aws_region + ) + logger.debug(f"AWS CodeArtifact token obtained for domain '{auth_cfg.aws_domain}' in region '{auth_cfg.aws_region}'") + return {"Authorization": f"Bearer {token}"} + + +def _aws_assume_role(auth_cfg: AuthConfig, cred_data: dict) -> dict: + if not auth_cfg.aws_role_arn: + raise ValueError("AWS assume_role requires awsRoleARN to be specified") + + _validate_user_pass_creds(cred_data, "AWS assume_role") + raise NotImplementedError("AWS assume_role auth is not yet implemented") + + +def _gcp_bearer(auth_cfg: AuthConfig, cred_data: dict) -> dict: + sa_key = cred_data.get(CRED_FIELD_SECRET) + if not sa_key: + raise ValueError("GCP service_account requires credential with 'secret' field containing SA JSON key") + + try: + json.loads(sa_key) + except json.JSONDecodeError: + raise ValueError("GCP service account key must be valid JSON") + + try: + from qubership_pipelines_common_library.v2.artifacts_finder.auth.gcp_credentials import GcpCredentialsProvider + except ImportError as e: + raise ValueError(f"GCP dependencies not available: {e}") + + creds = GcpCredentialsProvider().with_service_account_key( + service_account_key_content=sa_key, + ).get_credentials() + logger.debug(f"GCP token obtained for registry '{auth_cfg.gcp_reg_project}'") + return {"Authorization": f"Bearer {getattr(creds, GCP_TOKEN_ATTR)}"} + + +def _gcp_federation(auth_cfg: AuthConfig, cred_data: dict) -> dict: + if not auth_cfg.gcp_oidc: + raise ValueError("GCP federation requires gcpOIDC configuration") + + raise NotImplementedError("GCP federation (OIDC) auth is not yet implemented") + + +def _azure_oauth2(auth_cfg: AuthConfig, cred_data: dict) -> dict: + if not auth_cfg.azure_tenant_id: + raise ValueError("Azure OAuth2 requires azureTenantId") + + raise NotImplementedError("Azure OAuth2 auth is not yet implemented") + + +def _basic_auth(auth_cfg: AuthConfig, cred_data: dict) -> dict: + username, password = _validate_user_pass_creds(cred_data, "Basic auth") + token = base64.b64encode(f"{username}:{password}".encode()).decode() + return {"Authorization": f"Basic {token}"} + + +_PROVIDER_HANDLERS = { + (Provider.AWS, AUTH_METHOD_SECRET): _aws_bearer, + (Provider.AWS, AUTH_METHOD_ASSUME_ROLE): _aws_assume_role, + (Provider.GCP, AUTH_METHOD_SERVICE_ACCOUNT): _gcp_bearer, + (Provider.GCP, AUTH_METHOD_FEDERATION): _gcp_federation, + (Provider.AZURE, AUTH_METHOD_OAUTH2): _azure_oauth2, + (Provider.NEXUS, AUTH_METHOD_USER_PASS): _basic_auth, + (Provider.ARTIFACTORY, AUTH_METHOD_USER_PASS): _basic_auth, +} + + +def resolve_v2_auth_headers(registry: RegistryV2, env_creds: dict) -> Optional[dict]: + """Resolve V2 registry authConfig into HTTP Authorization headers. + Returns None for anonymous access.""" + auth_config = registry.maven_config.auth_config + if auth_config not in registry.auth_config: + raise ValueError( + f"AuthConfig '{auth_config}' not found in registry '{registry.name}'. " + f"Available: {list(registry.auth_config.keys())}") + + auth_cfg = registry.auth_config[auth_config] + + if auth_cfg.auth_method == AUTH_METHOD_ANONYMOUS: + logger.debug(f"Anonymous access for registry '{registry.name}'") + return None + + cred_data = _get_cred_data(auth_cfg.credentials_id, env_creds) + + handler = _PROVIDER_HANDLERS.get((auth_cfg.provider, auth_cfg.auth_method)) + if not handler: + raise ValueError( + f"Unsupported auth configuration (provider='{auth_cfg.provider.value}', " + f"authMethod='{auth_cfg.auth_method}') for registry '{registry.name}'") + + return handler(auth_cfg, cred_data) diff --git a/python/artifact-searcher/artifact_searcher/test_artifact.py b/python/artifact-searcher/artifact_searcher/test_artifact.py index e02a46435..da08f1010 100644 --- a/python/artifact-searcher/artifact_searcher/test_artifact.py +++ b/python/artifact-searcher/artifact_searcher/test_artifact.py @@ -2,11 +2,18 @@ import pytest from aiohttp import web +from unittest.mock import patch, Mock os.environ["DEFAULT_REQUEST_TIMEOUT"] = "0.2" # for test cases to run quicker from artifact_searcher.utils import models from artifact_searcher.artifact import check_artifact_async +from artifact_searcher.artifact import check_artifact +from artifact_searcher.utils.models import FileExtension +TEST_REPO = "https://repo.example.com/repository/" +GROUP_ID = "com.example" +ARTIFACT_ID = "demo" +VERSION = "1.0.0" class MockResponse: def __init__(self, status_code): @@ -71,9 +78,18 @@ def mock_get(url, *args, **kwargs): target_snapshot="repo", target_staging="repo", target_release="repo", - repository_domain_name=base_url, + repository_domain_name=base_url + ) + dcr_cfg = models.DockerConfig( + snapshot_uri="https://docker.example.com/snapshot", + staging_uri="https://docker.example.com/staging", + release_uri="https://docker.example.com/release", + group_uri="https://docker.example.com/group", + snapshot_repo_name="snapshot-repo", + staging_repo_name="staging-repo", + release_repo_name="release-repo", + group_name="test-group" ) - dcr_cfg = models.DockerConfig() reg = models.Registry( name="registry", maven_config=mvn_cfg, @@ -93,3 +109,72 @@ def mock_get(url, *args, **kwargs): sample_url = f"{base_url.rstrip('/repository/')}{index_path}repo/com/example/app/1.0.0-SNAPSHOT/app-1.0.0-20240702.123456-1.json" assert full_url == sample_url, f"expected: {sample_url}, received: {full_url}" + +@patch("artifact_searcher.artifact.requests.head") +@patch("artifact_searcher.artifact.create_artifact_name") +@patch("artifact_searcher.artifact.version_to_folder_name") +@patch("artifact_searcher.artifact.MavenConfig.is_nexus") +def test_artifact_found(mock_nexus, mock_folder, mock_name, mock_head): + + mock_nexus.return_value = False + mock_folder.return_value = VERSION + mock_name.return_value = "demo-1.0.0.zip" + + response = Mock() + response.status_code = 200 + mock_head.return_value = response + + result = check_artifact( + TEST_REPO, + GROUP_ID, + ARTIFACT_ID, + VERSION, + FileExtension.ZIP + ) + + assert result == ( + "https://repo.example.com/repository/com/example/demo/1.0.0/demo-1.0.0.zip" + ) + + +@patch("artifact_searcher.artifact.requests.head") +@patch("artifact_searcher.artifact.create_artifact_name") +@patch("artifact_searcher.artifact.version_to_folder_name") +@patch("artifact_searcher.artifact.MavenConfig.is_nexus") +def test_artifact_not_found(mock_nexus, mock_folder, mock_name, mock_head): + + mock_nexus.return_value = False + mock_folder.return_value = VERSION + mock_name.return_value = "demo-1.0.0.zip" + + response = Mock() + response.status_code = 404 + mock_head.return_value = response + + result = check_artifact( + TEST_REPO, + GROUP_ID, + ARTIFACT_ID, + VERSION, + FileExtension.ZIP + ) + + assert result is None + + +@patch("artifact_searcher.artifact.MavenConfig.is_nexus") +@patch("artifact_searcher.artifact.convert_nexus_repo_url_to_index_view") +def test_nexus_repo_conversion(mock_convert, mock_detect): + + mock_detect.return_value = True + mock_convert.return_value = "https://nexus.example.com/service/rest/repository/browse/" + + check_artifact( + TEST_REPO, + GROUP_ID, + ARTIFACT_ID, + VERSION, + FileExtension.ZIP + ) + + mock_convert.assert_called_once() diff --git a/python/artifact-searcher/artifact_searcher/test_auth_resolver.py b/python/artifact-searcher/artifact_searcher/test_auth_resolver.py new file mode 100644 index 000000000..affacc7dd --- /dev/null +++ b/python/artifact-searcher/artifact_searcher/test_auth_resolver.py @@ -0,0 +1,217 @@ +import base64 +import json +import sys +from unittest.mock import MagicMock, patch + +import pytest + +from artifact_searcher.auth_resolver import resolve_v2_auth_headers +from artifact_searcher.utils.models import AuthConfig, Provider, RegistryV2, MavenConfigV2 + + +@pytest.fixture +def env_creds(): + return { + "aws-cred": { + "data": { + "username": "AKIA_ACCESS_KEY", + "password": "secret_key_value" + } + }, + "gcp-cred": { + "data": { + "secret": '{"type": "service_account", "project_id": "my-project"}' + } + }, + "nexus-cred": { + "data": { + "username": "nexus_user", + "password": "nexus_pass" + } + }, + "artifactory-cred": { + "data": { + "username": "artifactory_user", + "password": "artifactory_pass" + } + }, + "anonymous-cred": { + "data": {} + } + } + + +@pytest.fixture +def base_registry_v2(): + return RegistryV2( + version="2.0", + name="test-registry", + auth_config={}, + maven_config=MavenConfigV2( + auth_config="test-auth", + repository_domain_name="https://registry.example.com" + ) + ) + + +class TestAnonymousAccess: + def test_nexus_anonymous_auth(self, base_registry_v2, env_creds): + base_registry_v2.auth_config = { + "test-auth": AuthConfig(provider=Provider.NEXUS, auth_method="anonymous") + } + + result = resolve_v2_auth_headers(base_registry_v2, env_creds) + + assert result is None + + def test_artifactory_anonymous_auth(self, base_registry_v2, env_creds): + base_registry_v2.auth_config = { + "test-auth": AuthConfig(provider=Provider.ARTIFACTORY, auth_method="anonymous") + } + + result = resolve_v2_auth_headers(base_registry_v2, env_creds) + + assert result is None + + def test_missing_authconfig_reference(self, base_registry_v2, env_creds): + base_registry_v2.maven_config.auth_config = "nonexistent" + base_registry_v2.auth_config = {} + + with pytest.raises(ValueError, match="AuthConfig 'nonexistent' not found"): + resolve_v2_auth_headers(base_registry_v2, env_creds) + + +class TestAWSAuthentication: + def test_aws_secret_success(self, base_registry_v2, env_creds, monkeypatch): + # Mock v1 library module for AWSCodeArtifactHelper + fake_utils_aws = MagicMock() + + monkeypatch.setitem(sys.modules, 'qubership_pipelines_common_library.v1.utils.utils_aws', fake_utils_aws) + + # Mock AWSCodeArtifactHelper.get_authorization_token static method + mock_helper_class = MagicMock() + mock_helper_class.get_authorization_token.return_value = "aws_token_123" + + fake_utils_aws.AWSCodeArtifactHelper = mock_helper_class + + base_registry_v2.auth_config = { + "aws-auth": AuthConfig( + credentials_id="aws-cred", + provider=Provider.AWS, + auth_method="secret", + aws_region="us-east-1", + aws_domain="my-domain" + ) + } + base_registry_v2.maven_config.auth_config = "aws-auth" + + result = resolve_v2_auth_headers(base_registry_v2, env_creds) + + assert result == {"Authorization": "Bearer aws_token_123"} + mock_helper_class.get_authorization_token.assert_called_once_with( + access_key="AKIA_ACCESS_KEY", + secret_key="secret_key_value", + domain="my-domain", + region_name="us-east-1" + ) + + def test_aws_missing_credentials(self, base_registry_v2, env_creds): + env_creds["aws-cred"]["data"] = {"username": "access_key"} + + base_registry_v2.auth_config = { + "aws-auth": AuthConfig( + credentials_id="aws-cred", + provider=Provider.AWS, + auth_method="secret", + aws_region="us-east-1", + aws_domain="my-domain" + ) + } + base_registry_v2.maven_config.auth_config = "aws-auth" + + with pytest.raises(ValueError, match="AWS secret auth requires both username and password in credentials"): + resolve_v2_auth_headers(base_registry_v2, env_creds) + + +class TestGCPAuthentication: + def test_gcp_service_account_success(self, base_registry_v2, env_creds, monkeypatch): + # Create fake module + fake_gcp_creds = MagicMock() + monkeypatch.setitem(sys.modules, 'qubership_pipelines_common_library.v2.artifacts_finder.auth.gcp_credentials', fake_gcp_creds) + + # Setup mock + mock_gcp_provider = MagicMock() + fake_gcp_creds.GcpCredentialsProvider = mock_gcp_provider + + mock_creds = MagicMock() + mock_creds.gcp_authorization_token = "gcp_token_123" + mock_gcp_provider.return_value.with_service_account_key.return_value.get_credentials.return_value = mock_creds + + base_registry_v2.auth_config = { + "gcp-auth": AuthConfig( + credentials_id="gcp-cred", + provider=Provider.GCP, + auth_method="service_account", + gcp_reg_project="my-project" + ) + } + base_registry_v2.maven_config.auth_config = "gcp-auth" + + result = resolve_v2_auth_headers(base_registry_v2, env_creds) + + assert result == {"Authorization": "Bearer gcp_token_123"} + mock_gcp_provider.return_value.with_service_account_key.assert_called_once_with( + service_account_key_content='{"type": "service_account", "project_id": "my-project"}' + ) + + def test_gcp_missing_secret(self, base_registry_v2, env_creds): + env_creds["gcp-cred"]["data"] = {} + + base_registry_v2.auth_config = { + "gcp-auth": AuthConfig( + credentials_id="gcp-cred", + provider=Provider.GCP, + auth_method="service_account" + ) + } + base_registry_v2.maven_config.auth_config = "gcp-auth" + + # Empty cred_data should raise error about missing secret + with pytest.raises(ValueError, match="GCP service_account requires credential with 'secret' field"): + resolve_v2_auth_headers(base_registry_v2, env_creds) + + def test_gcp_invalid_json(self, base_registry_v2, env_creds): + env_creds["gcp-cred"]["data"]["secret"] = "not valid json" + + base_registry_v2.auth_config = { + "gcp-auth": AuthConfig( + credentials_id="gcp-cred", + provider=Provider.GCP, + auth_method="service_account" + ) + } + base_registry_v2.maven_config.auth_config = "gcp-auth" + + with pytest.raises(ValueError, match="GCP service account key must be valid JSON"): + resolve_v2_auth_headers(base_registry_v2, env_creds) + + +class TestNexusArtifactoryAuthentication: + @pytest.mark.parametrize("provider,cred_id,username,password", [ + (Provider.NEXUS, "nexus-cred", "nexus_user", "nexus_pass"), + (Provider.ARTIFACTORY, "artifactory-cred", "artifactory_user", "artifactory_pass"), + ]) + def test_basic_auth_success(self, provider, cred_id, username, password, base_registry_v2, env_creds): + base_registry_v2.auth_config = { + "basic-auth": AuthConfig( + credentials_id=cred_id, + provider=provider, + auth_method="user_pass" + ) + } + base_registry_v2.maven_config.auth_config = "basic-auth" + + result = resolve_v2_auth_headers(base_registry_v2, env_creds) + + expected_token = base64.b64encode(f"{username}:{password}".encode()).decode() + assert result == {"Authorization": f"Basic {expected_token}"} diff --git a/python/artifact-searcher/artifact_searcher/utils/models.py b/python/artifact-searcher/artifact_searcher/utils/models.py index 8d76b6b87..c8832eb1f 100644 --- a/python/artifact-searcher/artifact_searcher/utils/models.py +++ b/python/artifact-searcher/artifact_searcher/utils/models.py @@ -1,7 +1,10 @@ from enum import Enum from typing import Optional +import base64 -from pydantic import BaseModel, ConfigDict, field_validator, Field, model_validator +import jsonschema +from envgenehelper.config_helper import get_regdef_v2_schema +from pydantic import BaseModel, ConfigDict, field_validator, Field from pydantic.alias_generators import to_camel import requests @@ -26,8 +29,6 @@ class MavenConfig(BaseSchema): snapshot_group: Optional[str] = "" release_group: Optional[str] = "" - is_nexus: bool = False - @field_validator('full_repository_url') def check_full_repository_url(cls, full_repository_url): if full_repository_url: @@ -38,20 +39,19 @@ def check_full_repository_url(cls, full_repository_url): def ensure_trailing_slash(cls, value): return value.rstrip("/") + "/" - @model_validator(mode="after") - def detect_nexus(self): - if not self.repository_domain_name.endswith("/repository/"): - return self - base = self.repository_domain_name[: -len("repository/")] + @staticmethod + def is_nexus(repository_domain_name: str) -> bool: + if not repository_domain_name.endswith("/repository/"): + return False + + base = repository_domain_name[: -len("repository/")] status_url = f"{base}service/rest/v1/status" + try: resp = requests.get(status_url, timeout=DEFAULT_REQUEST_TIMEOUT) - self.is_nexus = resp.status_code == 200 + return resp.status_code == 200 except Exception: - self.is_nexus = False - - return self - + return False class DockerConfig(BaseSchema): snapshot_uri: Optional[str] = "" @@ -102,6 +102,7 @@ class ArtifactInfo(BaseSchema): path: Optional[str] = "" local_path: Optional[str] = "" name: Optional[str] = "" + auth_headers: Optional[dict] = None class Registry(BaseSchema): @@ -115,15 +116,165 @@ class Registry(BaseSchema): helm_config: Optional[HelmConfig] = None helm_app_config: Optional[HelmAppConfig] = None + def resolve_auth(self, env_creds: Optional[dict] = None) -> Optional[dict]: + """Returns auth headers dict for V1 registries (basic auth). + Returns None if no credentials configured.""" + if not self.credentials_id or not env_creds: + return None + cred_data = env_creds.get(self.credentials_id, {}).get("data", {}) + username = cred_data.get("username") + password = cred_data.get("password") + if username and password: + token = base64.b64encode(f"{username}:{password}".encode()).decode() + return {"Authorization": f"Basic {token}"} + return None + + +REGDEF_V2_VERSION = "2.0" + + +class Provider(str, Enum): + NEXUS = "nexus" + ARTIFACTORY = "artifactory" + AWS = "aws" + GCP = "gcp" + AZURE = "azure" + + +class GcpOIDC(BaseSchema): + url: str = Field(alias="URL") + custom_params: Optional[list[dict[str, str]]] = None + + +class AuthConfig(BaseSchema): + credentials_id: Optional[str] = None + auth_type: Optional[str] = None + provider: Provider + auth_method: str + aws_region: Optional[str] = None + aws_domain: Optional[str] = None + aws_role_arn: Optional[str] = Field(default=None, alias="awsRoleARN") + aws_role_session_prefix: Optional[str] = None + gcp_oidc: Optional[GcpOIDC] = Field(default=None, alias="gcpOIDC") + gcp_reg_project: Optional[str] = None + gcp_reg_pool_id: Optional[str] = None + gcp_reg_provider_id: Optional[str] = None + gcp_reg_sa_email: Optional[str] = Field(default=None, alias="gcpRegSAEmail") + gcp_region: Optional[str] = None + azure_tenant_id: Optional[str] = None + azure_acr_resource: Optional[str] = Field(default=None, alias="azureACRResource") + azure_acr_name: Optional[str] = Field(default=None, alias="azureACRName") + azure_artifacts_resource: Optional[str] = None + + +class MavenConfigV2(BaseSchema): + auth_config: str + repository_domain_name: str + target_snapshot: Optional[str] = "" + target_staging: Optional[str] = "" + target_release: Optional[str] = "" + snapshot_group: Optional[str] = "" + release_group: Optional[str] = "" + + @field_validator('repository_domain_name') + def ensure_trailing_slash(cls, value): + return value.rstrip("/") + "/" + + +class DockerConfigV2(BaseSchema): + auth_config: str + snapshot_uri: str + staging_uri: str + release_uri: str + group_uri: str + snapshot_repo_name: str + staging_repo_name: str + release_repo_name: str + group_name: str + + +class GoConfigV2(BaseSchema): + auth_config: str + repository_domain_name: str + go_target_snapshot: str + go_target_release: str + go_proxy_repository: str + + +class RawConfigV2(BaseSchema): + auth_config: str + repository_domain_name: str + raw_target_snapshot: str + raw_target_release: str + raw_target_staging: str + raw_target_proxy: str + + +class NpmConfigV2(BaseSchema): + auth_config: str + repository_domain_name: str + npm_target_snapshot: str + npm_target_release: str + + +class HelmConfigV2(BaseSchema): + auth_config: str + repository_domain_name: str + helm_target_staging: str + helm_target_release: str + + +class HelmAppConfigV2(BaseSchema): + auth_config: str + repository_domain_name: str + helm_staging_repo_name: str + helm_release_repo_name: str + helm_group_repo_name: str + helm_dev_repo_name: str + + +class RegistryV2(BaseSchema): + name: str + version: str = REGDEF_V2_VERSION + auth_config: dict[str, AuthConfig] = {} + maven_config: MavenConfigV2 + docker_config: Optional[DockerConfigV2] = None + go_config: Optional[GoConfigV2] = None + raw_config: Optional[RawConfigV2] = None + npm_config: Optional[NpmConfigV2] = None + helm_config: Optional[HelmConfigV2] = None + helm_app_config: Optional[HelmAppConfigV2] = None + + def resolve_auth(self, env_creds: Optional[dict] = None) -> Optional[dict]: + """Returns auth headers dict for V2 registries (unified API with V1). + Returns None if anonymous or no credentials configured.""" + from artifact_searcher.auth_resolver import resolve_v2_auth_headers + return resolve_v2_auth_headers(self, env_creds or {}) + + +def parse_registry(data: dict) -> Registry | RegistryV2: + if data.get("version") == REGDEF_V2_VERSION or "authConfig" in data: + schema = get_regdef_v2_schema() + jsonschema.validate(instance=data, schema=schema) + return RegistryV2.model_validate(data) + return Registry.model_validate(data) + # artifact definition class Application(BaseSchema): name: str artifact_id: str group_id: str - registry: Registry + registry: Registry | RegistryV2 solution_descriptor: bool = False + @field_validator('registry', mode='before') + @classmethod + def parse_registry_field(cls, v): + if isinstance(v, dict): + return parse_registry(v) + return v + class FileExtension(str, Enum): ZIP = 'zip' diff --git a/python/artifact-searcher/pyproject.toml b/python/artifact-searcher/pyproject.toml index 131ac4b7c..5a11558ec 100644 --- a/python/artifact-searcher/pyproject.toml +++ b/python/artifact-searcher/pyproject.toml @@ -7,16 +7,20 @@ name = "artifact_searcher" version = "0.0.1" requires-python = "~=3.12" dependencies = [ - "pydantic~=2.10.6", - "requests~=2.32.3", - "deepdiff~=8.0.1", - "PyYAML~=6.0.2", - "responses~=0.25.7", - "aiohttp~=3.11.18", - "asyncio~=3.4.3", - "aioresponses~=0.7.8", - "pytest-asyncio~=1.0.0", - "pytest-aiohttp~=1.1.0" + "pydantic==2.10.6", + "requests==2.32.3", + "deepdiff==8.0.1", + "PyYAML==6.0.2", + "responses==0.25.7", + "aiohttp==3.11.18", + "asyncio==3.4.3", + "aioresponses==0.7.8", + "pytest-asyncio==1.0.0", + "pytest-aiohttp==1.1.0", + "boto3==1.39.4", + "google-auth==2.34.0", + "qubership-pipelines-common-library==2.0.3", + "jsonschema==4.24.1" ] [project.optional-dependencies] diff --git a/python/envgene/envgenehelper/__init__.py b/python/envgene/envgenehelper/__init__.py index 8adea78a9..cace6f169 100644 --- a/python/envgene/envgenehelper/__init__.py +++ b/python/envgene/envgenehelper/__init__.py @@ -2,7 +2,7 @@ from .yaml_helper import * from .file_helper import * from .business_helper import * -from .config_helper import get_envgene_config_yaml +from .config_helper import get_envgene_config_yaml, get_regdef_schema, get_regdef_v2_schema, validate_regdef_or_fail, get_regdef_schema_for_content from .json_helper import * from .collections_helper import * from .logger import logger diff --git a/python/envgene/envgenehelper/business_helper.py b/python/envgene/envgenehelper/business_helper.py index b8bb0aa5b..86879dd10 100644 --- a/python/envgene/envgenehelper/business_helper.py +++ b/python/envgene/envgenehelper/business_helper.py @@ -1,5 +1,6 @@ +from enum import auto, StrEnum import re -from dataclasses import dataclass, field +from dataclasses import InitVar, dataclass, field from os import getenv from pathlib import Path from typing import overload @@ -27,6 +28,8 @@ DEFAULT_PASSPORT_DIR_NAME = "cloud-passport" INV_GEN_CREDS_PATH = "Inventory/credentials/inventory_generation_creds.yml" +TEMPLATE_DIR_PATTERN = re.compile(r'/from_(\w+_)?template/') + def find_env_instances_dir(env_name, instances_dir): logger.debug(f"Searching for directory {env_name} in {instances_dir}") @@ -166,7 +169,8 @@ def getTemplateArtifactName(env_definition_yaml): return gav["artifact_id"] -def getEnvDefinition(env_dir): +def getEnvDefinition(env_dir = None): + env_dir = env_dir or get_current_env_dir_from_env_vars() env_definition_path = getEnvDefinitionPath(env_dir) if not check_file_exists(env_definition_path): raise ReferenceError(f"Environment definition for env {env_dir} is not found in {env_definition_path}") @@ -367,16 +371,36 @@ def find_cloud_name_from_passport(source_env_dir, all_instances_dir): else: return "" +class NamespaceRole(StrEnum): + COMMON = auto() + ORIGIN = auto() + PEER = auto() + +def get_namespace_role(ns_name: str, bgd_object: dict | None = None) -> NamespaceRole: + if not bgd_object: + bgd_object = get_bgd_object() + if not bgd_object: + return NamespaceRole.COMMON + if bgd_object['originNamespace']['name'] == ns_name: + return NamespaceRole.ORIGIN + if bgd_object['peerNamespace']['name'] == ns_name: + return NamespaceRole.PEER + return NamespaceRole.COMMON @dataclass class NamespaceFile: path: Path name: str = field(init=False) + postfix: str = field(init=False) definition_path: Path = field(init=False) + role: NamespaceRole = field(init=False) + bgd: InitVar[dict | None] = None - def __post_init__(self): + def __post_init__(self, bgd: dict | None): self.definition_path = self.path.joinpath('namespace.yml') self.name = openYaml(self.definition_path)['name'] + self.postfix = self.path.name + self.role = get_namespace_role(self.name, bgd) def get_namespaces_path(env_dir: Path | None = None) -> Path: @@ -385,17 +409,6 @@ def get_namespaces_path(env_dir: Path | None = None) -> Path: logger.debug(namespaces_path) return namespaces_path - -def get_namespaces(env_dir: Path | None = None) -> list[NamespaceFile]: - namespaces_path = get_namespaces_path(env_dir) - if not check_dir_exists(str(namespaces_path)): - return [] - namespace_paths = [p for p in namespaces_path.iterdir() if p.is_dir()] - namespaces = [NamespaceFile(path=p) for p in namespace_paths] - logger.debug(namespaces) - return namespaces - - def get_bgd_path(env_dir: Path | None = None) -> Path: env_dir = env_dir or get_current_env_dir_from_env_vars() bgd_path = env_dir.joinpath('bg_domain.yml') @@ -409,6 +422,31 @@ def get_bgd_object(env_dir: Path | None = None) -> CommentedMap: logger.debug(bgd_object) return bgd_object +def get_namespaces(env_dir: Path | None = None) -> list[NamespaceFile]: + namespaces_path = get_namespaces_path(env_dir) + if not check_dir_exists(str(namespaces_path)): + return [] + namespace_paths = [p for p in namespaces_path.iterdir() if p.is_dir()] + bgd = get_bgd_object(env_dir) + namespaces = [NamespaceFile(path=p, bgd=bgd) for p in namespace_paths] + logger.debug(namespaces) + return namespaces + +def get_template_dirs(base_dir: str | None = None) -> dict[NamespaceRole, str]: + base_dir = base_dir if base_dir else getenv_with_error('CI_PROJECT_DIR') + result = {} + result[NamespaceRole.COMMON] = f"{base_dir}/tmp/templates" + origin_template_path = f"{base_dir}/tmp/origin/templates" + if check_dir_exists(origin_template_path): + result[NamespaceRole.ORIGIN] = origin_template_path + peer_template_path = f"{base_dir}/tmp/peer/templates" + if check_dir_exists(peer_template_path): + result[NamespaceRole.PEER] = peer_template_path + return result + +def is_from_template_dir(file_path: str) -> bool: + return bool(TEMPLATE_DIR_PATTERN.search(file_path)) + -def parse_env_names(full_env_names: str): - return full_env_names.split("\n") +def get_sboms_dir(work_dir) -> Path: + return Path(work_dir) / "sboms" diff --git a/python/envgene/envgenehelper/collections_helper.py b/python/envgene/envgenehelper/collections_helper.py index ef301fb8a..557e9d382 100644 --- a/python/envgene/envgenehelper/collections_helper.py +++ b/python/envgene/envgenehelper/collections_helper.py @@ -2,6 +2,8 @@ from pprint import pformat from .yaml_helper import yaml import copy +from .logger import logger +from enum import Enum def merge_lists(list1, list2) : if len(list2) > 0 : @@ -13,14 +15,30 @@ def merge_lists(list1, list2) : def is_primitive(obj): return isinstance(obj, primitives) -def dump_as_yaml_format(collection) : - if collection and isinstance(collection, dict): - tmp = copy.deepcopy(collection) +def _convert_enums(obj): + if isinstance(obj, Enum): + return obj.value + if isinstance(obj, dict): + return { + _convert_enums(k): _convert_enums(v) + for k, v in obj.items() + } + if isinstance(obj, list): + return [_convert_enums(v) for v in obj] + if isinstance(obj, tuple): + return tuple(_convert_enums(v) for v in obj) + if isinstance(obj, set): + return {_convert_enums(v) for v in obj} + return obj + + +def dump_as_yaml_format(collection): + converted = _convert_enums(collection) + if converted and isinstance(converted, dict): stream = StringIO() - yaml.dump(tmp, stream) + yaml.dump(converted, stream) return stream.getvalue() - else: - return pformat(collection) + return pformat(converted) def get_merged_param_value(key, source_dict, override_dict): if isinstance(override_dict[key], dict): @@ -95,3 +113,41 @@ def _compare_dicts_recurse(source: object, target: object, path: DictPath, diff_ elif source != target: diff_paths.append(path.copy()) +def split_multi_value_param(param: str)-> list[str]: + + if not param: + return [] + + param = param.strip() + if not param: + return [] + + has_comma = ',' in param + has_semicolon = ';' in param + has_space = ' ' in param + has_newline = '\n' in param + + delimiter_count = sum([has_comma, has_semicolon, has_space, has_newline]) + + if delimiter_count > 1: + raise ValueError( + "Invalid input: use only ONE delimiter type (comma, semicolon, space, or newline)" + ) + + if has_comma: + logger.info(f"env names {param} has comma as delimiter. splitting it") + parts = param.split(',') + elif has_semicolon: + logger.info(f"env names {param} has semicolon as delimiter. splitting it") + parts = param.split(';') + elif has_space: + logger.info(f"env names {param} has space as delimiter. splitting it") + parts = param.split() + elif has_newline: + logger.info(f"env names {param} has newline as delimiter. splitting it") + parts = param.splitlines() + else: + return [param] + + return [p.strip() for p in parts if p.strip()] + diff --git a/python/envgene/envgenehelper/config_helper.py b/python/envgene/envgenehelper/config_helper.py index 716494539..46f795be5 100644 --- a/python/envgene/envgenehelper/config_helper.py +++ b/python/envgene/envgenehelper/config_helper.py @@ -1,8 +1,10 @@ +from importlib.resources import files from os import getenv, path import json from pathlib import Path from envgenehelper import openYaml, get_empty_yaml, getenv_with_error +from envgenehelper.yaml_helper import validate_yaml_by_scheme_or_fail import jsonschema from .logger import logger @@ -14,6 +16,31 @@ FERNET_ID = "Fernet" SOPS_ID = "SOPS" +REGDEF_V2_VERSION = "2.0" + +def get_regdef_schema() -> dict: + """Load RegDef V1 schema from package resources""" + return json.loads(files("envgenehelper").joinpath("schemas/regdef.schema.json").read_text(encoding="utf-8")) + + +def get_regdef_v2_schema() -> dict: + """Load RegDef V2 schema from package resources""" + return json.loads(files("envgenehelper").joinpath("schemas/regdef-v2.schema.json").read_text(encoding="utf-8")) + + +def get_regdef_schema_for_content(content: dict) -> dict: + """Get the appropriate schema (V1 or V2) based on registry definition content""" + if content.get("version") == REGDEF_V2_VERSION or "authConfig" in content: + return get_regdef_v2_schema() + return get_regdef_schema() + + +def validate_regdef_or_fail(yaml_file_path: str): + """Validate a registry definition YAML file (V1 or V2) by path""" + content = openYaml(yaml_file_path) + schema = get_regdef_schema_for_content(content) + validate_yaml_by_scheme_or_fail(yaml_file_path=yaml_file_path, input_schema_content=schema) + def get_schema(schema_name): schemas_folder = "schemas" diff --git a/python/envgene/envgenehelper/constants.py b/python/envgene/envgenehelper/constants.py index 136ab51f5..f7f2f3c10 100644 --- a/python/envgene/envgenehelper/constants.py +++ b/python/envgene/envgenehelper/constants.py @@ -9,3 +9,5 @@ "bg_domain.yml", "composite_structure.yml", ] + +CI_JOB_ARTIFACT_MAX_SIZE_MB = 1200 # 80% from limit 1.5 diff --git a/python/envgene/envgenehelper/creds_helper.py b/python/envgene/envgenehelper/creds_helper.py index d8c5c8c6c..2334a5505 100644 --- a/python/envgene/envgenehelper/creds_helper.py +++ b/python/envgene/envgenehelper/creds_helper.py @@ -1,7 +1,8 @@ import re from pathlib import Path -from envgenehelper import crypt, getenv_with_error +from envgenehelper import crypt, getenv_with_error, get_env_instances_dir, findAllYamlsInDir, openYaml, getEnvCredentialsPath +from envgenehelper.errors import ValidationError from .logger import logger @@ -219,3 +220,56 @@ def get_cred_config(): base_dir = getenv_with_error('CI_PROJECT_DIR') cred_config = crypt.decrypt_file(Path(f"{base_dir}/configuration/credentials/credentials.yml")) return cred_config + + +def validate_creds(creds_path: str = ""): + if not creds_path: + environment_name = getenv_with_error('ENVIRONMENT_NAME') + cluster_name = getenv_with_error('CLUSTER_NAME') + instances_dir = getenv_with_error('INSTANCES_DIR') + env_dir = get_env_instances_dir(environment_name, cluster_name, instances_dir) + creds_path = Path(getEnvCredentialsPath(env_dir)).parent + + credsErrors = [] + credsYamls = findAllYamlsInDir(creds_path) + logger.info(f"Starting validation of credentials") + for credListPath in credsYamls: + credListYaml = openYaml(credListPath) + for key, value in credListYaml.items() : + errorCheck = check_cred_value(key, value) + if errorCheck : + credsErrors.append(errorCheck) + if len(credsErrors) > 0: + errorMessage = "Error while validating credentials: \n" + for err in credsErrors: + errorMessage += f"\t{err}\n" + raise ValidationError(errorMessage) + + logger.info(f"Validation of credentials is completed") + + +def check_cred_value(credId, credValue) -> str: + result = "" + type = credValue["type"] + data = credValue["data"] + match type: + case _ if type == CRED_TYPE_USERPASS: + if is_envgenenullvalue(data["username"]) or is_envgenenullvalue(data["password"]): + result = f"credId: {credId} - username or password is not set" + case _ if type == CRED_TYPE_SECRET: + if is_envgenenullvalue(data["secret"]): + result = f"credId: {credId} - secret is not set" + case _ if type == CRED_TYPE_VAULT: + if is_envgenenullvalue(data["secretId"]): + result = f"credId: {credId} - secretId is not set" + case _: + result = "" + return result + + +def is_envgenenullvalue(value: object) -> bool: + if not isinstance(value, str): + return False + if value.lower() == "envgenenullvalue": + return True + return False diff --git a/python/envgene/envgenehelper/crypt.py b/python/envgene/envgenehelper/crypt.py index ef9bb6d4f..83b72f199 100644 --- a/python/envgene/envgenehelper/crypt.py +++ b/python/envgene/envgenehelper/crypt.py @@ -7,6 +7,7 @@ from .yaml_helper import openYaml, get_empty_yaml from .file_helper import check_file_exists, get_files_with_filter from .logger import logger +from .collections_helper import split_multi_value_param from .crypt_backends.fernet_handler import crypt_Fernet, extract_value_Fernet, is_encrypted_Fernet from .crypt_backends.sops_handler import crypt_SOPS, extract_value_SOPS, is_encrypted_SOPS @@ -117,7 +118,7 @@ def get_all_necessary_cred_files() -> set[str]: if env_names == "env_template_test": logger.info("Running in env_template_test mode") return get_files_with_filter(BASE_DIR, is_cred_file) - env_names_list = env_names.split("\n") + env_names_list = split_multi_value_param(env_names) sources = set() sources.add("configuration") diff --git a/python/envgene/envgenehelper/file_helper.py b/python/envgene/envgenehelper/file_helper.py index 32f7174a8..6b547612c 100644 --- a/python/envgene/envgenehelper/file_helper.py +++ b/python/envgene/envgenehelper/file_helper.py @@ -3,6 +3,7 @@ import re import shutil import tarfile +import time import zipfile from typing import Callable from pathlib import Path @@ -198,18 +199,22 @@ def findFiles(fileList: list[Path], pattern, notPattern="", additionalRegexpPatt for filePath in fileList: # this ensures that pattern matching works correctly on both Windows (\) and Unix (/) file_path_posix = Path(filePath).as_posix() + pattern_posix = Path(pattern).as_posix() if pattern else "" + not_pattern_posix = Path(notPattern).as_posix() if notPattern else "" if ( - pattern in file_path_posix - and (notPattern == "" or notPattern not in file_path_posix) + pattern_posix in file_path_posix + and (not_pattern_posix == "" or not_pattern_posix not in file_path_posix) and (additionalRegexpPattern == "" or re.match(additionalRegexpPattern, file_path_posix)) and (additionalRegexpNotPattern == "" or not re.match(additionalRegexpNotPattern, file_path_posix)) ): result.append(filePath) logger.debug( - f"Path {filePath} match pattern: {pattern} or notPattern: {notPattern} or additionalPattern: {additionalRegexpPattern}") + f"Path {filePath} match pattern: {pattern_posix} or notPattern: {not_pattern_posix} " + f"or additionalPattern: {additionalRegexpPattern}") else: logger.debug( - f"Path {filePath} doesn't match pattern: {pattern} or notPattern: {notPattern} or additionalPattern: {additionalRegexpPattern}") + f"Path {filePath} doesn't match pattern: {pattern_posix} or notPattern: {not_pattern_posix} " + f"or additionalPattern: {additionalRegexpPattern}") return result @@ -258,3 +263,43 @@ def cleanup_dir(path: str): def is_dir_empty(dir_path): dir_path = Path(dir_path) return dir_path.exists() and dir_path.is_dir() and not any(dir_path.iterdir()) + + +def cleanup_dir_by_size(dir_path, max_size_mb): + dir_path = Path(dir_path) + if not dir_path.exists(): + logger.warning(f"Path does not exist: {dir_path}") + return + + mb = 1024 * 1024 + max_size = max_size_mb * mb + + files = [Path(f) for f in findAllFilesInDir(dir_path, "")] + total = sum(f.stat().st_size for f in files) + total_mb = total / mb + + if total <= max_size: + logger.info(f"Directory size {total_mb:.2f} mb within limit {max_size_mb} mb") + return + + logger.info(f"Directory size {total_mb:.2f} mb exceeds limit {max_size_mb} mb, deleting all files in {dir_path}") + for file in files: + logger.info(f"Removing file: {file}") + deleteFileIfExists(file) + + +def cleanup_dir_by_age(dir_path, keep_last: int): + dir_path = Path(dir_path) + + if not dir_path.exists(): + logger.warning(f"Path does not exist: {dir_path}") + return + + files = [f for f in dir_path.iterdir() if f.is_file()] + files.sort(key=lambda f: f.stat().st_mtime, reverse=True) + + if len(files) > keep_last: + logger.info(f"Only {keep_last} files will remain in {dir_path}") + for old_file in files[keep_last:]: + logger.info(f"Removing file: {old_file}") + deleteFileIfExists(old_file) diff --git a/python/envgene/envgenehelper/models.py b/python/envgene/envgenehelper/models.py new file mode 100644 index 000000000..af7c83730 --- /dev/null +++ b/python/envgene/envgenehelper/models.py @@ -0,0 +1,22 @@ +from enum import Enum + +from pydantic import BaseModel, Field + + +class TemplateVersionUpdateMode(str, Enum): + PERSISTENT = "PERSISTENT" + TEMPORARY = "TEMPORARY" + + @classmethod + def _missing_(cls, value): + if isinstance(value, str): + value = value.upper() + for m in cls: + if m.value == value: + return m + return None + + +class SbomRetentionConfig(BaseModel): + enabled: bool = Field(default=False) + keep_versions_per_app: int = Field(default=10, ge=0) diff --git a/schemas/regdef-v2.schema.json b/python/envgene/envgenehelper/schemas/regdef-v2.schema.json similarity index 67% rename from schemas/regdef-v2.schema.json rename to python/envgene/envgenehelper/schemas/regdef-v2.schema.json index 7f1a30249..7fcd62784 100644 --- a/schemas/regdef-v2.schema.json +++ b/python/envgene/envgenehelper/schemas/regdef-v2.schema.json @@ -43,8 +43,7 @@ "required": [ "version", "name", - "mavenConfig", - "dockerConfig" + "mavenConfig" ], "definitions": { "AuthConfig": { @@ -66,7 +65,9 @@ "enum": [ "aws", "azure", - "gcp" + "gcp", + "nexus", + "artifactory" ] }, "authMethod": { @@ -77,7 +78,8 @@ "federation", "service_account", "oauth2", - "user_pass" + "user_pass", + "anonymous" ] }, "awsRegion": { @@ -125,6 +127,9 @@ "gcpRegSAEmail": { "type": "string" }, + "gcpRegion": { + "type": "string" + }, "azureTenantId": { "type": "string" }, @@ -139,7 +144,152 @@ } }, "required": [ - "credentialsId" + "provider", + "authMethod" + ], + "allOf": [ + { + "if": { + "properties": { + "authMethod": { + "not": { + "const": "anonymous" + } + } + } + }, + "then": { + "required": ["credentialsId"] + } + }, + { + "if": { + "properties": { + "provider": { + "const": "aws" + }, + "authMethod": { + "const": "secret" + } + }, + "required": ["provider", "authMethod"] + }, + "then": { + "required": ["awsRegion"] + } + }, + { + "if": { + "properties": { + "provider": { + "const": "aws" + }, + "authMethod": { + "const": "assume_role" + } + }, + "required": ["provider", "authMethod"] + }, + "then": { + "required": ["awsRoleARN"] + } + }, + { + "if": { + "properties": { + "provider": { + "const": "gcp" + }, + "authMethod": { + "const": "federation" + } + }, + "required": ["provider", "authMethod"] + }, + "then": { + "required": ["gcpOIDC"] + } + }, + { + "if": { + "properties": { + "provider": { + "const": "nexus" + } + } + }, + "then": { + "properties": { + "authMethod": { + "enum": ["user_pass", "anonymous"] + } + } + } + }, + { + "if": { + "properties": { + "provider": { + "const": "artifactory" + } + } + }, + "then": { + "properties": { + "authMethod": { + "enum": ["user_pass", "anonymous"] + } + } + } + }, + { + "if": { + "properties": { + "provider": { + "const": "aws" + } + } + }, + "then": { + "properties": { + "authMethod": { + "enum": ["secret", "assume_role", "anonymous"] + } + } + } + }, + { + "if": { + "properties": { + "provider": { + "const": "gcp" + } + } + }, + "then": { + "properties": { + "authMethod": { + "enum": ["federation", "service_account", "anonymous"] + } + } + } + }, + { + "if": { + "properties": { + "provider": { + "const": "azure" + } + } + }, + "then": { + "properties": { + "authMethod": { + "enum": ["oauth2", "anonymous"] + } + } + } + } ] }, "DockerConfig": { @@ -175,6 +325,7 @@ } }, "required": [ + "authConfig", "groupName", "groupUri", "releaseRepoName", @@ -212,12 +363,8 @@ } }, "required": [ - "releaseGroup", - "repositoryDomainName", - "snapshotGroup", - "targetRelease", - "targetSnapshot", - "targetStaging" + "authConfig", + "repositoryDomainName" ] }, "GoConfig": { @@ -241,6 +388,7 @@ } }, "required": [ + "authConfig", "repositoryDomainName", "goTargetSnapshot", "goTargetRelease", @@ -271,6 +419,7 @@ } }, "required": [ + "authConfig", "repositoryDomainName", "rawTargetSnapshot", "rawTargetRelease", @@ -296,6 +445,7 @@ } }, "required": [ + "authConfig", "repositoryDomainName", "npmTargetSnapshot", "npmTargetRelease" @@ -319,6 +469,7 @@ } }, "required": [ + "authConfig", "repositoryDomainName", "helmTargetStaging", "helmTargetRelease" @@ -348,6 +499,7 @@ } }, "required": [ + "authConfig", "repositoryDomainName", "helmStagingRepoName", "helmReleaseRepoName", diff --git a/python/envgene/envgenehelper/schemas/regdef.schema.json b/python/envgene/envgenehelper/schemas/regdef.schema.json new file mode 100644 index 000000000..7b20d25af --- /dev/null +++ b/python/envgene/envgenehelper/schemas/regdef.schema.json @@ -0,0 +1,224 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "credentialsId": { + "type": "string" + }, + "mavenConfig": { + "$ref": "#/definitions/MavenConfig" + }, + "dockerConfig": { + "$ref": "#/definitions/DockerConfig" + }, + "goConfig": { + "$ref": "#/definitions/GoConfig" + }, + "rawConfig": { + "$ref": "#/definitions/RawConfig" + }, + "npmConfig": { + "$ref": "#/definitions/NpmConfig" + }, + "helmConfig": { + "$ref": "#/definitions/HelmConfig" + }, + "helmAppConfig": { + "$ref": "#/definitions/HelmAppConfig" + } + }, + "required": [ + "name", + "credentialsId", + "mavenConfig", + "dockerConfig" + ], + "definitions": { + "mapString": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "DockerConfig": { + "type": "object", + "additionalProperties": false, + "properties": { + "snapshotUri": { + "type": "string" + }, + "stagingUri": { + "type": "string" + }, + "releaseUri": { + "type": "string" + }, + "groupUri": { + "type": "string" + }, + "snapshotRepoName": { + "type": "string" + }, + "stagingRepoName": { + "type": "string" + }, + "releaseRepoName": { + "type": "string" + }, + "groupName": { + "type": "string" + } + }, + "required": [ + "groupName", + "groupUri", + "releaseRepoName", + "releaseUri", + "snapshotRepoName", + "snapshotUri", + "stagingRepoName", + "stagingUri" + ] + }, + "MavenConfig": { + "type": "object", + "additionalProperties": false, + "properties": { + "repositoryDomainName": { + "type": "string" + }, + "fullRepositoryUrl": { + "type": "string" + }, + "targetSnapshot": { + "type": "string" + }, + "targetStaging": { + "type": "string" + }, + "targetRelease": { + "type": "string" + }, + "snapshotGroup": { + "type": "string" + }, + "releaseGroup": { + "type": "string" + } + }, + "required": [ + "fullRepositoryUrl", + "releaseGroup", + "repositoryDomainName", + "snapshotGroup", + "targetRelease", + "targetSnapshot", + "targetStaging" + ] + }, + "GoConfig": { + "type": "object", + "additionalProperties": false, + "properties": { + "goTargetSnapshot": { + "type": "string" + }, + "goTargetRelease": { + "type": "string" + }, + "goProxyRepository": { + "type": "string" + } + }, + "required": [ + "goTargetSnapshot", + "goTargetRelease", + "goProxyRepository" + ] + }, + "RawConfig": { + "type": "object", + "additionalProperties": false, + "properties": { + "rawTargetSnapshot": { + "type": "string" + }, + "rawTargetRelease": { + "type": "string" + }, + "rawTargetStaging": { + "type": "string" + }, + "rawTargetProxy": { + "type": "string" + } + }, + "required": [ + "rawTargetSnapshot", + "rawTargetRelease", + "rawTargetStaging", + "rawTargetProxy" + ] + }, + "NpmConfig": { + "type": "object", + "additionalProperties": false, + "properties": { + "npmTargetSnapshot": { + "type": "string" + }, + "npmTargetRelease": { + "type": "string" + } + }, + "required": [ + "npmTargetSnapshot", + "npmTargetRelease" + ] + }, + "HelmConfig": { + "type": "object", + "additionalProperties": false, + "properties": { + "helmTargetStaging": { + "type": "string" + }, + "helmTargetRelease": { + "type": "string" + } + }, + "required": [ + "helmTargetStaging", + "helmTargetRelease" + ] + }, + "HelmAppConfig": { + "type": "object", + "additionalProperties": false, + "properties": { + "helmStagingRepoName": { + "type": "string" + }, + "helmReleaseRepoName": { + "type": "string" + }, + "helmGroupRepoName": { + "type": "string" + }, + "helmDevRepoName": { + "type": "string" + } + }, + "required": [ + "helmStagingRepoName", + "helmReleaseRepoName", + "helmGroupRepoName", + "helmDevRepoName" + ] + } + } +} diff --git a/python/envgene/envgenehelper/test_collections.py b/python/envgene/envgenehelper/test_collections.py index 550cc28ae..77e7ecc6f 100644 --- a/python/envgene/envgenehelper/test_collections.py +++ b/python/envgene/envgenehelper/test_collections.py @@ -1,4 +1,5 @@ from .collections_helper import * +import pytest def convert_list_elements_to_strings_in_place(lst): for i in range(len(lst)): @@ -59,3 +60,38 @@ def test_compare_dicts(): assert len(s_diff) == len(diff) # no duplicates assert s_diff == set(expected_arr) and removed == [], f"Failed Test 8: {diff}, {removed}" + +@pytest.mark.parametrize( + "value, expected", + [ + ("env01", ["env01"]), + ("env01,env02", ["env01", "env02"]), + ("env01;env02", ["env01", "env02"]), + ("env01 env02", ["env01", "env02"]), + ("env01\nenv02", ["env01", "env02"]), + (" env01 env02 ", ["env01", "env02"]), + ("app1:1.0", ["app1:1.0"]), + ("app1:1.0;app2:2.0", ["app1:1.0", "app2:2.0"]), + ("app1:1.0;app2:2.0", ["app1:1.0", "app2:2.0"]), + ("app1:1.0;app2:2.0", ["app1:1.0", "app2:2.0"]), + ("app1:1.0;app2:2.0", ["app1:1.0", "app2:2.0"]), + ("", []), + (" ", []), + ], +) +def test_split_multi_value_param_valid(value, expected): + assert split_multi_value_param(value) == expected + + +@pytest.mark.parametrize( + "value", + [ + "env01, env02", + "env01; env02", + "env01\nenv02 env03", + "env01,env02;env03", + ], +) +def test_split_multi_value_param_invalid(value): + with pytest.raises(ValueError, match=r"use only ONE delimiter type"): + split_multi_value_param(value) \ No newline at end of file diff --git a/python/envgene/envgenehelper/test_helpers/test_helpers.py b/python/envgene/envgenehelper/test_helpers/test_helpers.py index cc71e5356..634ac42de 100644 --- a/python/envgene/envgenehelper/test_helpers/test_helpers.py +++ b/python/envgene/envgenehelper/test_helpers/test_helpers.py @@ -20,7 +20,7 @@ def clean_test_dir(path: str | Path) -> Path: return path @staticmethod - def compare_dirs_content(source_dir, target_dir) -> tuple[list[str], list[str], dict[str,str] | list, list]: + def compare_dirs_content(source_dir, target_dir) -> tuple[list[str], list[str], dict[str, str] | list, list]: source_map = {Path(f).name: f for f in get_all_files_in_dir(source_dir)} target_map = {Path(f).name: f for f in get_all_files_in_dir(target_dir)} @@ -40,10 +40,12 @@ def compare_dirs_content(source_dir, target_dir) -> tuple[list[str], list[str], file1 = os.path.join(source_dir, file) file2 = os.path.join(target_dir, file) with open(file1, 'r') as f1, open(file2, 'r') as f2: - verbose_diff_list = difflib.unified_diff(f1.readlines(), f2.readlines(), fromfile=file1, tofile=file2, lineterm='') + verbose_diff_list = difflib.unified_diff(f1.readlines(), f2.readlines(), fromfile=file1, + tofile=file2, lineterm='') diff_list = [] for line in verbose_diff_list: - if (line.startswith('+') and not line.startswith('+++')) or (line.startswith('-') and not line.startswith('---')): + if (line.startswith('+') and not line.startswith('+++')) or ( + line.startswith('-') and not line.startswith('---')): diff_list.append(line) mismach_dict[file] = ''.join(diff_list) logger.error(f"Diff for {file}:\n{''.join(verbose_diff_list)}") @@ -52,7 +54,8 @@ def compare_dirs_content(source_dir, target_dir) -> tuple[list[str], list[str], return extra_files, missing_files, mismatch, errors @staticmethod - def assert_dirs_content(source_dir, target_dir, check_for_missing_files=False, check_for_extra_files=False, expected_mismatch:dict[str, str] | None=None): + def assert_dirs_content(source_dir, target_dir, check_for_missing_files=False, check_for_extra_files=False, + expected_mismatch: dict[str, str] | None = None): extra_files, missing_files, mismatch, errors = TestHelpers.compare_dirs_content(source_dir, target_dir) if check_for_extra_files: @@ -77,3 +80,8 @@ def create_fake_zip(files: dict[str, str] = None) -> bytes: zf.writestr(filename, content) return zip_buffer.getvalue() + @staticmethod + def create_file(path: Path, size=1024, mtime=None): + path.write_bytes(b"x" * size) + if mtime is not None: + os.utime(path, (mtime, mtime)) diff --git a/python/envgene/envgenehelper/yaml_helper.py b/python/envgene/envgenehelper/yaml_helper.py index 029dcef7e..345abb5f5 100644 --- a/python/envgene/envgenehelper/yaml_helper.py +++ b/python/envgene/envgenehelper/yaml_helper.py @@ -8,13 +8,13 @@ import jschon_tools import jsonschema import ruyaml +from jsonschema import RefResolver from ruyaml import CommentedMap, CommentedSeq from ruyaml.scalarstring import DoubleQuotedScalarString, LiteralScalarString from .file_helper import * from .json_helper import openJson from .logger import logger -from jsonschema import RefResolver def create_yaml_processor(is_safe=False) -> ruyaml.main.YAML: @@ -294,6 +294,26 @@ def beautifyYaml(file_path, schema_path="", header_text="", allign_comments=Fals alignYamlFileComments(file_path) +def find_yaml_file(dir_path: Path, search_name: str, recursively: bool = False) -> Path | None: + + if not dir_path.exists(): + return None + + if recursively: + for root, _, files in os.walk(dir_path): + for f in files: + if f.endswith((".yml", ".yaml")): + if Path(f).stem == search_name: + return Path(root) / f + else: + for entry in os.scandir(dir_path): + if entry.is_file() and entry.name.endswith((".yml", ".yaml")): + if Path(entry.name).stem == search_name: + return Path(entry.path) + + return None + + def findYamls(dir, pattern, notPattern="", additionalRegexpPattern="", additionalRegexpNotPattern=""): fileList = findAllYamlsInDir(dir) return findFiles(fileList, pattern, notPattern, additionalRegexpPattern, additionalRegexpNotPattern) @@ -433,15 +453,17 @@ def convert_ordereddict_to_dict(obj): return obj -def find_all_yaml_files_by_stem(path: str): - file_paths = [] - for ext in ["yaml", "yml"]: - file_path = Path(f"{path}.{ext}") - if file_path.exists(): - file_paths.append(file_path) - return file_paths +def find_files_by_basename(path: str, extensions_priority: tuple[str] = ("yml", "yaml")) -> list[Path]: + base_path = Path(path) + found_files: list[Path] = [] + + for ext in extensions_priority: + candidate = base_path.with_suffix(f".{ext}") + if candidate.exists(): + found_files.append(candidate) + return found_files jschon.create_catalog('2020-12') yaml = create_yaml_processor() -safe_yaml = create_yaml_processor(is_safe=True) +safe_yaml = create_yaml_processor(is_safe=True) \ No newline at end of file diff --git a/python/envgene/pyproject.toml b/python/envgene/pyproject.toml index 3be997722..adef7cbe0 100644 --- a/python/envgene/pyproject.toml +++ b/python/envgene/pyproject.toml @@ -1,7 +1,26 @@ +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + + +[project] +name = "envgenehelper" +version = "0.0.1" +requires-python = "~=3.12" + +dependencies = [ + "pydantic~=2.10.6" +] + + +[tool.setuptools.package-data] +envgenehelper = ["schemas/*.json"] + [tool.black] line-length = 120 skip-string-normalization = true + [tool.mypy] warn_return_any = true warn_unused_configs = true diff --git a/schemas/artifact-definition-v2.schema.json b/schemas/artifact-definition-v2.schema.json index dde0fd065..8a61e73fd 100644 --- a/schemas/artifact-definition-v2.schema.json +++ b/schemas/artifact-definition-v2.schema.json @@ -93,7 +93,7 @@ "credentialsId": { "type": "string", "title": "Credentials ID", - "description": "Pointer to the EnvGene Credential object. Depending on authType, it can be access key (username) + secret (password) for longLived. Credential with this ID must be located in /configuration/credentials/credentials.yml" + "description": "Not used in case of authMethod: anonymous. Pointer to the EnvGene Credential object. Depending on authType, it can be access key (username) + secret (password) for longLived. Credential with this ID must be located in /configuration/credentials/credentials.yml" }, "authType": { "type": "string", @@ -109,10 +109,12 @@ "enum": [ "aws", "azure", - "gcp" + "gcp", + "nexus", + "artifactory" ], "title": "Provider", - "description": "Public cloud registry type. Used in case of public cloud registries" + "description": "Registry type" }, "authMethod": { "type": "string", @@ -122,10 +124,11 @@ "federation", "service_account", "oauth2", - "user_pass" + "user_pass", + "anonymous" ], "title": "Authentication Method", - "description": "In case of non-cloud public registries, user_pass is used. In case of public cloud registries valid values depend on provider: aws: secret or assume_role, gcp: federation or service_account, azure: oauth2" + "description": "In case of non-cloud public registries, user_pass is used. In case of public cloud registries valid values depend on provider: nexus: user_pass or anonymous, artifactory: user_pass or anonymous, aws: secret, assume_role or anonymous, gcp: federation, service_account or anonymous, azure: oauth2 or anonymous" }, "awsRegion": { "type": "string", @@ -216,7 +219,136 @@ } }, "required": [ - "credentialsId" + "provider", + "authMethod" + ], + "allOf": [ + { + "if": { + "properties": { + "authMethod": { + "not": { + "const": "anonymous" + } + } + } + }, + "then": { + "required": ["credentialsId"] + } + }, + { + "if": { + "properties": { + "provider": { + "const": "aws" + }, + "authMethod": { + "const": "assume_role" + } + }, + "required": ["provider", "authMethod"] + }, + "then": { + "required": ["awsRoleARN"] + } + }, + { + "if": { + "properties": { + "provider": { + "const": "gcp" + }, + "authMethod": { + "const": "federation" + } + }, + "required": ["provider", "authMethod"] + }, + "then": { + "required": ["gcpOIDC"] + } + }, + { + "if": { + "properties": { + "provider": { + "const": "nexus" + } + } + }, + "then": { + "properties": { + "authMethod": { + "enum": ["user_pass", "anonymous"] + } + } + } + }, + { + "if": { + "properties": { + "provider": { + "const": "artifactory" + } + } + }, + "then": { + "properties": { + "authMethod": { + "enum": ["user_pass", "anonymous"] + } + } + } + }, + { + "if": { + "properties": { + "provider": { + "const": "aws" + } + } + }, + "then": { + "properties": { + "authMethod": { + "enum": ["secret", "assume_role", "anonymous"] + } + } + } + }, + { + "if": { + "properties": { + "provider": { + "const": "gcp" + } + } + }, + "then": { + "properties": { + "authMethod": { + "enum": ["federation", "service_account", "anonymous"] + } + } + } + }, + { + "if": { + "properties": { + "provider": { + "const": "azure" + } + } + }, + "then": { + "properties": { + "authMethod": { + "enum": ["oauth2", "anonymous"] + } + } + } + } ] }, "MavenConfig": { @@ -228,7 +360,7 @@ "authConfig": { "type": "string", "title": "Authentication Config Reference", - "description": "Pointer to authentication config described in authConfig section. Cannot be set if anonymous access is used" + "description": "Pointer to authentication config described in authConfig section" }, "repositoryDomainName": { "type": "string", @@ -286,12 +418,8 @@ } }, "required": [ - "repositoryDomainName", - "targetSnapshot", - "targetStaging", - "targetRelease", - "snapshotGroup", - "releaseGroup" + "authConfig", + "repositoryDomainName" ] } } diff --git a/schemas/config.schema.json b/schemas/config.schema.json index c8f27ab93..371d097fa 100644 --- a/schemas/config.schema.json +++ b/schemas/config.schema.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "title": "Generic Configuration", - "description": "Configuration for the for cryptographic operations, artifact discovery, and cloud passport decryption", + "description": "Configuration for cryptographic operations, artifact discovery, SBOM retention, and cloud passport decryption", "additionalProperties": true, "properties": { "crypt": { @@ -57,6 +57,36 @@ "cmdb", "local" ] + }, + "sbom_retention": { + "type": "object", + "title": "SBOM Retention Configuration", + "description": "Configuration for automatic SBOM file cleanup to manage repository size. Triggers only when repository reaches 1200 GB threshold.", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "title": "Enable SBOM Retention", + "description": "Enable or disable SBOM retention cleanup", + "default": false, + "examples": [ + true, + false + ] + }, + "keep_versions_per_app": { + "type": "integer", + "title": "Versions to Keep", + "description": "Number of latest versions to keep per application. Used only when enabled is true.", + "minimum": 1, + "default": 10, + "examples": [ + 5, + 10, + 15 + ] + } + } } } } diff --git a/schemas/custom-params.schema.json b/schemas/custom-params.schema.json new file mode 100644 index 000000000..a62e943cb --- /dev/null +++ b/schemas/custom-params.schema.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://qubership.org/schemas/custom-params.schema.json", + "title": "CUSTOM_PARAMS", + "description": "Schema for the CUSTOM_PARAMS instance pipeline parameter.", + "type": "object", + "additionalProperties": false, + "properties": { + "deployment": { + "type": "object", + "description": "Overrides applied to the deployment and cleanup context of Effective Set" + }, + "runtime": { + "type": "object", + "description": "Overrides applied to the runtime context of Effective Set" + } + } +} diff --git a/schemas/env-definition.schema.json b/schemas/env-definition.schema.json index 7febd9a7f..7e4fbf01f 100644 --- a/schemas/env-definition.schema.json +++ b/schemas/env-definition.schema.json @@ -5,6 +5,12 @@ "description": "Configuration for the environment, Environment Inventory", "additionalProperties": true, "properties": { + "metadata": { + "type": "object", + "title": "Metadata", + "description": "Optional metadata map for the environment definition. Structure is not specified. Used only by Colly.", + "additionalProperties": true + }, "inventory": { "type": "object", "title": "Environment definition for Inventory", diff --git a/scripts/bg_manage/bg_manage.py b/scripts/bg_manage/bg_manage.py index be4246302..206280824 100644 --- a/scripts/bg_manage/bg_manage.py +++ b/scripts/bg_manage/bg_manage.py @@ -4,7 +4,7 @@ from enum import auto, Enum from pathlib import Path -from envgenehelper.business_helper import get_current_env_dir_from_env_vars, getenv_with_error, get_namespaces, get_bgd_object +from envgenehelper.business_helper import get_current_env_dir_from_env_vars, getenv_with_error, get_namespaces, get_bgd_object, NamespaceRole from envgenehelper.file_helper import deleteFileIfExists from envgenehelper.yaml_helper import openYaml from envgenehelper import logger, writeYamlToFile @@ -106,10 +106,10 @@ def get_current_state() -> Pair: continue multiple_state_files_err_msg = f"Multiple state files found in {ENV_PATH}" - if role == "origin": + if role == NamespaceRole.ORIGIN: if origin_state != S.NONE: raise ValueError(multiple_state_files_err_msg + " for 'origin'") origin_state = state_enum - elif role == "peer": + elif role == NamespaceRole.PEER: if peer_state != S.NONE: raise ValueError(multiple_state_files_err_msg + " for 'peer'") peer_state = state_enum diff --git a/scripts/build_env/appregdef_render.py b/scripts/build_env/appregdef_render.py index 9cded545f..7f4551c70 100644 --- a/scripts/build_env/appregdef_render.py +++ b/scripts/build_env/appregdef_render.py @@ -1,44 +1,43 @@ -import shutil +from envgenehelper import * +from envgenehelper.models import TemplateVersionUpdateMode from env_template.process_env_template import process_env_template -from envgenehelper import * from render_config_env import EnvGenerator - if __name__ == '__main__': template_version = process_env_template() - + cluster_name = getenv_with_error("CLUSTER_NAME") env_name = getenv_with_error("ENVIRONMENT_NAME") base_dir = getenv_with_error('CI_PROJECT_DIR') instances_dir = getenv_with_error("INSTANCES_DIR") - + output_dir = f"{base_dir}/environments" render_dir = f"/tmp/render/{env_name}" - templates_dir = f"{base_dir}/tmp/templates" - + templates_dirs = get_template_dirs() + env_dir = get_env_instances_dir(env_name, cluster_name, instances_dir) cloud_passport_file_path = find_cloud_passport_definition(env_dir, instances_dir) - + render_context_vars = { "cluster_name": cluster_name, "output_dir": output_dir, "current_env_dir": render_dir, - "templates_dir": templates_dir, + "templates_dirs": templates_dirs, "cloud_passport_file_path": cloud_passport_file_path, "env_instances_dir": env_dir } - + render_context = EnvGenerator() render_context.process_app_reg_defs(env_name, render_context_vars) - + for dir_name in ["AppDefs", "RegDefs"]: src = Path(render_dir) / dir_name dst = Path(env_dir) / dir_name - + if dst.exists(): shutil.rmtree(dst) if src.exists(): shutil.move(src, dst) - - update_generated_versions(env_dir, BUILD_ENV_TAG, template_version) \ No newline at end of file + + update_generated_versions(env_dir, BUILD_ENV_TAG, template_version[NamespaceRole.COMMON]) diff --git a/scripts/build_env/build_env.py b/scripts/build_env/build_env.py index 9caae2f1d..dd40aec66 100644 --- a/scripts/build_env/build_env.py +++ b/scripts/build_env/build_env.py @@ -1,15 +1,10 @@ -import os -import sys -import copy -import json import yaml -import re -import pathlib from envgenehelper import * -from resource_profiles import processResourceProfiles -from schema_validation import checkEnvSpecificParametersBySchema + from cloud_passport import process_cloud_passport -from pathlib import Path +from resource_profiles import collect_resource_profiles, override_by_env_specific_profiles, has_valid_profile_name, \ + update_profile_name +from schema_validation import checkEnvSpecificParametersBySchema # const GENERATED_HEADER = "The contents of this file is generated from template artifact: %s.\nContents will be overwritten by next generation.\nPlease modify this contents only for development purposes or as workaround." @@ -26,25 +21,39 @@ def find_namespaces(dir): return result -def processFileList(mask, dict, dirPointer): - fileList = list(dirPointer.rglob(mask)) - for f in fileList: - filePath = str(f) - # envSpecific = false will be update later during templates parsing - key = extractNameFromFile(filePath) - if key in dict: - dict[extractNameFromFile(filePath)].append({"filePath": filePath, "envSpecific": False}) - else: - dict[extractNameFromFile(filePath)] = [{"filePath": filePath, "envSpecific": False}] - return dict +def _get_excluded_dirs_for_role(role: NamespaceRole, origin_template_exists: bool, peer_template_exists: bool) -> list: + common_dir = 'from_template' + origin_dir = 'from_origin_template' + peer_dir = 'from_peer_template' + if role == NamespaceRole.ORIGIN and origin_template_exists: + return [common_dir, peer_dir] + elif role == NamespaceRole.PEER and peer_template_exists: + return [common_dir, origin_dir] + else: + return [origin_dir, peer_dir] -def createParamsetsMap(dir): +def create_paramset_map(dir: str, role: NamespaceRole, + origin_template_exists: bool, peer_template_exists: bool) -> dict: + excluded_dirs = _get_excluded_dirs_for_role(role, origin_template_exists, peer_template_exists) result = {} - dirPointer = pathlib.Path(dir) + dir_pointer = pathlib.Path(dir) masks = ["*.json", "*.yml", "*.yaml", "*.j2"] + for mask in masks: - result = processFileList(mask, result, dirPointer) + for f in dir_pointer.rglob(mask): + file_path = str(f) + if any(excluded_dir in file_path for excluded_dir in excluded_dirs): + continue + key = extractNameFromFile(file_path) + entry = {"filePath": file_path, "envSpecific": False} + if key in result: + result[key].append(entry) + else: + result[key] = [entry] + + logger.info(f"Created {role.name}-specific paramset map: excluded dirs {excluded_dirs}, " + f"origin_template_exists={origin_template_exists}, peer_template_exists={peer_template_exists}") logger.debug(f'List of {dir} paramsets: \n %s', dump_as_yaml_format(result)) return result @@ -180,14 +189,15 @@ def findEnvDefinitionFromTemplatePath(templatePath, env_instances_dir=None): def sort_paramsets_with_same_name(entries: list[dict]) -> list[dict]: - # strict order processing paramsets template -> instance + # Strict order processing paramsets template -> cluster -> instance + # Lower sort keys are processed first, later values override earlier ones def sort_key(e): path = e["filePath"] - if "from_template" in path: - return 1, path - elif "from_instance" in path: + if "from_instance" in path: return 2, path - return 0, path + elif is_from_template_dir(path): + return 0, path + return 1, path return sorted(entries, key=sort_key) @@ -264,7 +274,7 @@ def convertParameterSetsToParameters(templatePath, paramsTemplate, paramsetsTag, else: paramSetAppParams = [] paramsetDefinitionComment = "paramset: " + paramSetName + " version: " + str( - paramSetVersion) + " source: " + ("template" if "from_template" in paramSetFile else "instance") + paramSetVersion) + " source: " + ("template" if is_from_template_dir(paramSetFile) else "instance") # process parameters in ParamSet for k in paramSetParameters: # get value with potential merge of dicts @@ -407,8 +417,7 @@ def processTemplate(templatePath, templateName, env_instances_dir, schema_path, env_instances_dir) templateContent["technicalConfigurationParameterSets"] = [] # preparing map for needed resource profiles - if "profile" in templateContent and templateContent["profile"] and "name" in templateContent["profile"] and \ - templateContent["profile"]["name"]: + if has_valid_profile_name(templateContent): rpName = templateContent["profile"]["name"] resource_profiles_map[templateName] = rpName writeYamlToFile(templatePath, templateContent) @@ -456,14 +465,23 @@ def process_additional_template_parameters(render_env_dir, source_env_dir, all_i logger.info(f"No shared templates variables are defined in: {envDefinitionPath}") -def getTemplateNameFromNamespacePath(namespacePath): - path = pathlib.Path(namespacePath) - return path.parent.name - - def build_env(env_name, env_instances_dir, parameters_dir, env_template_dir, resource_profiles_dir, - env_specific_resource_profile_map, all_instances_dir, render_context): - paramset_map = createParamsetsMap(parameters_dir) + env_specific_resource_profile_map, all_instances_dir, render_context, templates_dirs=None): + # Check which role-specific templates were downloaded + templates_dirs = templates_dirs or {} + origin_template_exists = NamespaceRole.ORIGIN in templates_dirs + peer_template_exists = NamespaceRole.PEER in templates_dirs + logger.info(f"Templates dirs: {list(templates_dirs.keys())}, " + f"origin_exists={origin_template_exists}, peer_exists={peer_template_exists}") + + # Create role-specific paramset maps + origin_paramset_map = create_paramset_map(parameters_dir, NamespaceRole.ORIGIN, + origin_template_exists, peer_template_exists) + peer_paramset_map = create_paramset_map(parameters_dir, NamespaceRole.PEER, + origin_template_exists, peer_template_exists) + common_paramset_map = create_paramset_map(parameters_dir, NamespaceRole.COMMON, + origin_template_exists, peer_template_exists) + env_dir = env_template_dir + "/" + env_name logger.info(f"Env name: {env_name}") logger.info(f"Env dir: {env_dir}") @@ -482,7 +500,7 @@ def build_env(env_name, env_instances_dir, parameters_dir, env_template_dir, res # pathes tenantTemplatePath = env_dir + "/tenant.yml" cloudTemlatePath = env_dir + "/cloud.yml" - namespaceTemplates = find_namespaces(env_dir) + namespaces = get_namespaces(Path(env_dir)) # env specific parameters map - will be filled with env specific parameters during template processing env_specific_parameters_map = {} env_specific_parameters_map["namespaces"] = {} @@ -501,7 +519,7 @@ def build_env(env_name, env_instances_dir, parameters_dir, env_template_dir, res "cloud", env_instances_dir, cloud_schema, - paramset_map, + common_paramset_map, env_specific_parameters_map["cloud"], resource_profiles_map=needed_resource_profiles_map, header_text=generated_header_text, @@ -514,7 +532,7 @@ def build_env(env_name, env_instances_dir, parameters_dir, env_template_dir, res "cloud", env_instances_dir, cloud_schema, - paramset_map, + common_paramset_map, env_specific_parameters_map["cloud"], resource_profiles_map=needed_resource_profiles_map, header_text=generated_header_text, @@ -522,25 +540,51 @@ def build_env(env_name, env_instances_dir, parameters_dir, env_template_dir, res # process namespaces template_namespace_names = [] - # iterate through namespace definitions and create namespace parameters - for templatePath in namespaceTemplates: - logger.info(f"Processing namespace: {templatePath}") - templateName = getTemplateNameFromNamespacePath(templatePath) - template_namespace_names.append(templateName) - initParametersStructure(env_specific_parameters_map["namespaces"], templateName) + for ns in namespaces: + logger.info(f"Processing namespace: {ns.definition_path}") + template_namespace_names.append(ns.postfix) + initParametersStructure(env_specific_parameters_map['namespaces'], ns.postfix) + + if ns.role == NamespaceRole.ORIGIN: + ns_paramset_map = origin_paramset_map + elif ns.role == NamespaceRole.PEER: + ns_paramset_map = peer_paramset_map + else: + ns_paramset_map = common_paramset_map processTemplate( - templatePath, - templateName, + ns.definition_path, + ns.postfix, env_instances_dir, namespace_schema, - paramset_map, - env_specific_parameters_map["namespaces"][templateName], + ns_paramset_map, + env_specific_parameters_map['namespaces'][ns.postfix], resource_profiles_map=needed_resource_profiles_map, - header_text=generated_header_text) - + header_text=generated_header_text, + ) logger.info(f"EnvSpecific parameters are: \n{dump_as_yaml_format(env_specific_parameters_map)}") checkEnvSpecificParametersBySchema(env_dir, env_specific_parameters_map, template_namespace_names) # process resource profiles - processResourceProfiles(env_dir, resource_profiles_dir, profiles_schema, needed_resource_profiles_map, - env_specific_resource_profile_map, render_context, header_text=generated_header_text) + result_profiles_dir = Path(f"{env_dir}/Profiles") + all_profiles = collect_resource_profiles(result_profiles_dir, resource_profiles_dir, profiles_schema, + needed_resource_profiles_map, render_context) + override_profile_map = override_by_env_specific_profiles(all_profiles, env_specific_resource_profile_map, + render_context) + + if override_profile_map: + for profile_key, profile_file_path in override_profile_map.items(): + all_profiles[profile_key] = profile_file_path + profile_name = openYaml(profile_file_path, {}).get("name") + + if profile_key == 'cloud': + update_profile_name(cloudTemlatePath, profile_name) + + for ns in namespaces: + if profile_key == ns.postfix: + update_profile_name(ns.definition_path, profile_name) + + for profile_key, profile_file_path in all_profiles.items(): + logger.info(f"Copying '{profile_key}' to resulting directory '{result_profiles_dir}'") + copy_path(profile_file_path, f"{result_profiles_dir}/") + resulting_profile_path = result_profiles_dir / Path(profile_file_path).name + beautifyYaml(resulting_profile_path, profiles_schema, generated_header_text) diff --git a/scripts/build_env/cloud_passport.py b/scripts/build_env/cloud_passport.py index 05c5e5e00..f4c11f8fa 100644 --- a/scripts/build_env/cloud_passport.py +++ b/scripts/build_env/cloud_passport.py @@ -45,6 +45,8 @@ def process_cloud_definition(cloudPassportYaml, env_dir, comment) : process_and_update_key("maasUrl", cloudYaml["maasConfig"], "MAAS_SERVICE_ADDRESS", maasPassportYaml, comment) process_and_update_key("maasInternalAddress", cloudYaml["maasConfig"], "MAAS_INTERNAL_ADDRESS", maasPassportYaml, comment) del cloudPassportYaml["maas"] + else: + store_value_to_yaml(cloudYaml["maasConfig"], "enable", False) if "vault" in cloudPassportYaml : vaultPassportYaml = cloudPassportYaml["vault"] vaultUrl = vaultPassportYaml["VAULT_ADDR"] @@ -57,6 +59,8 @@ def process_cloud_definition(cloudPassportYaml, env_dir, comment) : store_value_to_yaml(cloudYaml["vaultConfig"], "enable", False, comment) store_value_to_yaml(cloudYaml["vaultConfig"], "url", "", comment) del cloudPassportYaml["vault"] + else: + store_value_to_yaml(cloudYaml["vaultConfig"], "enable", False) if "dbaas" in cloudPassportYaml : dbaasPassportYaml = cloudPassportYaml["dbaas"] if "dbaasConfigs" not in cloudYaml or len(cloudYaml["dbaasConfigs"]) != 1 : @@ -68,6 +72,9 @@ def process_cloud_definition(cloudPassportYaml, env_dir, comment) : process_and_update_key("apiUrl", dbaasConfigYaml, "API_DBAAS_ADDRESS", dbaasPassportYaml, comment) process_and_update_key("aggregatorUrl", dbaasConfigYaml, "DBAAS_AGGREGATOR_ADDRESS", dbaasPassportYaml, comment) del cloudPassportYaml["dbaas"] + else: + for cfg in cloudYaml["dbaasConfigs"]: + store_value_to_yaml(cfg, "enable", False) if "consul" in cloudPassportYaml : consulPassportYaml = cloudPassportYaml["consul"] consulConfigYaml = cloudYaml["consulConfig"] @@ -76,8 +83,10 @@ def process_cloud_definition(cloudPassportYaml, env_dir, comment) : process_and_update_key("publicUrl", consulConfigYaml, "CONSUL_PUBLIC_URL", consulPassportYaml, comment) process_and_update_key("internalUrl", consulConfigYaml, "CONSUL_URL", consulPassportYaml, comment) # CONSUL_ENABLED variable should be both in consul section and in deploy parameters - store_value_to_yaml(cloudYaml["deployParameters"], "CONSUL_ENABLED", f"{consulConfigYaml['enabled']}".lower(), comment) + store_value_to_yaml(cloudYaml["deployParameters"], "CONSUL_ENABLED", consulConfigYaml['enabled'], comment) del cloudPassportYaml["consul"] + else: + store_value_to_yaml(cloudYaml["consulConfig"], "enabled", False) # adding rest of cloud passport parameters to cloud deploy parameters logger.debug(f"Rest of params from cloud passport are: \n{dump_as_yaml_format(cloudPassportYaml)}") mergeDeployParametersFromPassport(cloudPassportYaml, cloudYaml, comment) diff --git a/scripts/build_env/create_credentials.py b/scripts/build_env/create_credentials.py index 10b5eb3c8..441e2a550 100644 --- a/scripts/build_env/create_credentials.py +++ b/scripts/build_env/create_credentials.py @@ -1,5 +1,4 @@ -import re -import sys +from pathlib import Path import os from envgenehelper import * @@ -159,18 +158,26 @@ def mergeAndSaveYaml(yamlPath, newCreds) : logger.info("%s credentials created" % count) writeYamlToFile(yamlPath, credsYaml) -def findSharedCredentials(cred_name, env_dir, instances_dir): - logger.debug(f"Searching for cred file {cred_name} from {env_dir} to {instances_dir}") - credFiles = findResourcesBottomTop(env_dir, instances_dir, f"/{cred_name}") - if len(credFiles) == 1: - yamlPath = credFiles[0] - logger.info(f"Shared credentials for {cred_name} found in: {yamlPath}") - return yamlPath - elif len(credFiles) > 1: - logger.error(f"Duplicate shared credentials with key {cred_name} found in {instances_dir}: \n\t" + ",\n\t".join(str(x) for x in credFiles)) - raise ReferenceError(f"Duplicate shared credentials with key {cred_name} found. See logs above.") - else: - raise ReferenceError(f"Shared credentials with key {cred_name} not found in {instances_dir}") + +def findSharedCredentials(cred_name, env_dir, instances_dir) -> Path: + levels = [ + Path(env_dir) / "Inventory", + Path(env_dir).parent, + Path(instances_dir), + ] + + cred_dir_names = ["credentials", "Credentials", "shared-credentials"] + + shared_cred_paths = [level / name for level in levels for name in cred_dir_names] + + for p in shared_cred_paths: + found_path = find_yaml_file(p, cred_name, recursively=True) + if found_path: + logger.info(f"Shared credentials with key '{cred_name}' found in '{found_path}'") + return found_path + + raise FileNotFoundError(f"Shared credentials with key '{cred_name}' not found.") + def mergeSharedCreds(credYamlPath, envDir, instancesDir) : inventoryYaml = getEnvDefinition(envDir) diff --git a/scripts/build_env/env_template/process_env_template.py b/scripts/build_env/env_template/process_env_template.py index d4c91ff80..e1314cc37 100644 --- a/scripts/build_env/env_template/process_env_template.py +++ b/scripts/build_env/env_template/process_env_template.py @@ -7,23 +7,29 @@ from artifact_searcher.utils.models import FileExtension, Credentials, Registry, Application from env_template.template_testing import run_env_test_setup from envgenehelper import getEnvDefinition, fetch_cred_value, getAppDefinitionPath -from envgenehelper import openYaml, getenv_with_error, logger +from envgenehelper import openYaml, getenv_with_error, logger, get_or_create_nested_yaml_attribute from envgenehelper import unpack_archive, get_cred_config, check_dir_exist_and_create +from envgenehelper.business_helper import NamespaceRole +from envgenehelper.yaml_helper import get_nested_yaml_attribute_or_fail from render_config_env import render_obj_by_context, Context -ARTIFACT_DEST = f"{tempfile.gettempdir()}/artifact.zip" +def parse_artifact_appver(env_definition: dict, attribute_str: str) -> list[str]: + try: + get_nested_yaml_attribute_or_fail(env_definition, attribute_str) + exists = True + except ValueError: + exists = False + appver = str(get_or_create_nested_yaml_attribute(env_definition, attribute_str, "")) -def parse_artifact_appver(env_definition: dict) -> [str, str]: - artifact_appver = env_definition.get('envTemplate', {}).get('artifact') - if not artifact_appver: - raise ValueError(f"Environment template artifact is empty or missing from env_definition: {env_definition}") - logger.info(f"Environment template artifact version: {artifact_appver}") - return artifact_appver.split(':') + if exists and not appver: + raise ValueError(f"{attribute_str} is empty or missing from env_definition: {env_definition}") + logger.info(f"Artifact version in {attribute_str}: {appver}") + return appver.split(":") -def get_registry_creds(registry: Registry) -> Credentials: - cred_config = render_creds() + +def get_registry_creds(registry: Registry, cred_config: dict) -> Credentials: cred_id = registry.credentials_id if cred_id: username = cred_config[cred_id]['data'].get('username') @@ -57,48 +63,46 @@ def validate_url(url, group_id, artifact_id, version): # logic resolving template by artifact definition -def resolve_artifact_new_logic(env_definition: dict, template_dest: str) -> str: - app_name, app_version = parse_artifact_appver(env_definition) - - base_dir = getenv_with_error('CI_PROJECT_DIR') - artifact_path = getAppDefinitionPath(base_dir, app_name) - if not artifact_path: - raise FileNotFoundError(f"No artifact definition file found for {app_name} with .yaml or .yml extension") - app_def = Application.model_validate(openYaml(artifact_path)) - cred = get_registry_creds(app_def.registry) +async def resolve_artifact_new_logic(app_def: Application, app_version: str, template_dest: str, auth_headers: dict = None) -> str: template_url = None resolved_version = app_version - dd_artifact_info = asyncio.run(artifact.check_artifact_async(app_def, FileExtension.JSON, app_version, cred)) + dd_artifact_info = await artifact.check_artifact_async(app_def, FileExtension.JSON, app_version, auth_headers=auth_headers) if dd_artifact_info: logger.info("Loading environment template artifact info from deployment descriptor...") - dd_url, dd_repo = dd_artifact_info + dd_url, (repo_name, _) = dd_artifact_info logger.info(f"Resolved deployment descriptor URL: {dd_url}") if "-SNAPSHOT" in app_version: resolved_version = extract_snapshot_version(dd_url, app_version) - dd_config = artifact.download_json_content(dd_url, cred) + dd_config = artifact.download_json_content(dd_url, auth_headers=auth_headers) group_id, artifact_id, version = parse_maven_coord_from_dd(dd_config) logger.info( f"Parsed maven coordinates: group_id={group_id}, artifact_id={artifact_id}, version={version} from dd") if not all([group_id, artifact_id, version]): raise ValueError(f"Invalid maven coordinates from deployment descriptor {dd_url}") - repo_url = dd_config.get("configurations", [{}])[0].get("maven_repository") or dd_repo - template_url = artifact.check_artifact(repo_url, group_id, artifact_id, version, FileExtension.ZIP, cred) + repo_url = dd_config.get("configurations", [{}])[0].get("maven_repository") + + if not repo_url: + repo_url = f"{app_def.registry.maven_config.repository_domain_name}{repo_name}" + logger.info(f"building repo url from the repo name : {repo_url}") + template_url = artifact.check_artifact(repo_url, group_id, artifact_id, version, FileExtension.ZIP, + auth_headers=auth_headers) validate_url(template_url, group_id, artifact_id, version) else: logger.info("Loading environment template artifact from zip directly...") group_id, artifact_id, version = app_def.group_id, app_def.artifact_id, app_version - artifact_info = asyncio.run(artifact.check_artifact_async(app_def, FileExtension.ZIP, app_version, cred)) + artifact_info = await artifact.check_artifact_async(app_def, FileExtension.ZIP, app_version, auth_headers=auth_headers) if artifact_info: template_url, _ = artifact_info validate_url(template_url, group_id, artifact_id, version) if "-SNAPSHOT" in app_version: resolved_version = extract_snapshot_version(template_url, app_version) logger.info(f"Environment template url has been resolved: {template_url}") - artifact.download(template_url, ARTIFACT_DEST, cred) - unpack_archive(ARTIFACT_DEST, template_dest) + artifact_dest = tempfile.mkstemp(suffix='.zip')[1] + artifact.download(template_url, artifact_dest, auth_headers=auth_headers) + unpack_archive(artifact_dest, template_dest) return resolved_version @@ -112,7 +116,7 @@ def render_creds() -> dict: # logic resolving template by exact coordinates and repo, deprecated -def resolve_artifact_old_logic(env_definition: dict, template_dest: str) -> str: +async def resolve_artifact_old_logic(env_definition: dict, template_dest: str, cred_config: dict, registry_dict: dict) -> str: template_artifact = env_definition['envTemplate']['templateArtifact'] artifact_info = template_artifact['artifact'] @@ -123,61 +127,89 @@ def resolve_artifact_old_logic(env_definition: dict, template_dest: str) -> str: repo_type = template_artifact['templateRepository'] registry_name = template_artifact['registry'] - registry_dict = openYaml( - Path(f"{getenv_with_error('CI_PROJECT_DIR')}/configuration/registry.yml")) # another registry model registry = registry_dict[registry_name] repo_url = registry.get(repo_type) dd_repo_url = registry.get(dd_repo_type) - cred_config = render_creds() repository_username = fetch_cred_value(registry.get("username"), cred_config) repository_password = fetch_cred_value(registry.get("password"), cred_config) cred = Credentials(username=repository_username, password=repository_password) + auth_headers = artifact.credentials_to_headers(cred) if cred.username and cred.password else None template_url = None resolved_version = dd_version - dd_url = artifact.check_artifact(dd_repo_url, group_id, artifact_id, dd_version, FileExtension.JSON, cred) + dd_url = artifact.check_artifact(dd_repo_url, group_id, artifact_id, dd_version, FileExtension.JSON, auth_headers=auth_headers) if dd_url: logger.info(f"Deployment descriptor url for environment template has been resolved: {dd_url}") if "-SNAPSHOT" in dd_version: resolved_version = extract_snapshot_version(dd_url, dd_version) - dd_config = artifact.download_json_content(dd_url, cred) + dd_config = artifact.download_json_content(dd_url, auth_headers=auth_headers) group_id, artifact_id, version = parse_maven_coord_from_dd(dd_config) logger.info( f"Parsed maven coordinates from dd: group_id={group_id}, artifact_id={artifact_id}, version={version}") if not all([group_id, artifact_id, version]): raise ValueError(f"Invalid maven coordinates from deployment descriptor {dd_url}") - template_url = artifact.check_artifact(repo_url, group_id, artifact_id, version, FileExtension.ZIP, cred) + template_url = artifact.check_artifact(repo_url, group_id, artifact_id, version, FileExtension.ZIP, auth_headers=auth_headers) validate_url(template_url, group_id, artifact_id, version) else: logger.info("Loading environment template artifact from zip directly...") - template_url = artifact.check_artifact(repo_url, group_id, artifact_id, dd_version, FileExtension.ZIP, cred) + template_url = artifact.check_artifact(repo_url, group_id, artifact_id, dd_version, FileExtension.ZIP, auth_headers=auth_headers) validate_url(template_url, group_id, artifact_id, dd_version) if "-SNAPSHOT" in dd_version: resolved_version = extract_snapshot_version(template_url, dd_version) logger.info(f"Environment template url has been resolved: {template_url}") - artifact.download(template_url, ARTIFACT_DEST, cred) - unpack_archive(ARTIFACT_DEST, template_dest) + artifact_dest = tempfile.mkstemp(suffix='.zip')[1] + artifact.download(template_url, artifact_dest, auth_headers=auth_headers) + unpack_archive(artifact_dest, template_dest) return resolved_version -def process_env_template() -> str: +def process_env_template() -> dict: env_template_test = os.getenv("ENV_TEMPLATE_TEST", "").lower() == "true" if env_template_test: run_env_test_setup() - project_dir = getenv_with_error('CI_PROJECT_DIR') - template_dest = f"{project_dir}/tmp" - cluster = getenv_with_error("CLUSTER_NAME") - environment = getenv_with_error("ENVIRONMENT_NAME") - env_dir = Path(f"{project_dir}/environments/{cluster}/{environment}") - env_definition = getEnvDefinition(env_dir) - check_dir_exist_and_create(template_dest) + env_definition = getEnvDefinition() - if 'artifact' in env_definition.get('envTemplate', {}): - logger.info("Use template resolving new logic") - return resolve_artifact_new_logic(env_definition, template_dest) - else: - logger.info("Use template resolving old logic") - return resolve_artifact_old_logic(env_definition, template_dest) + appvers = { + NamespaceRole.COMMON: parse_artifact_appver(env_definition, 'envTemplate.artifact'), + NamespaceRole.ORIGIN: parse_artifact_appver(env_definition, 'envTemplate.bgNsArtifacts.origin'), + NamespaceRole.PEER: parse_artifact_appver(env_definition, 'envTemplate.bgNsArtifacts.peer'), + } + + tasks = {} + project_dir = getenv_with_error('CI_PROJECT_DIR') + cred_config = render_creds() + + for template_type, appver in appvers.items(): + if template_type == NamespaceRole.COMMON: + template_dest = f'{project_dir}/tmp' + else: + template_dest = f'{project_dir}/tmp/{template_type}' + + if not (len(appver) >= 2 and bool(appver[0]) and bool(appver[1])): + if template_type != NamespaceRole.COMMON: + continue + registry_dict = openYaml(Path(f"{project_dir}/configuration/registry.yml")) + + logger.info('Using template resolving old logic') + tasks[template_type] = resolve_artifact_old_logic(env_definition, template_dest, cred_config, registry_dict) + continue + + app_name, app_version = appver[0], appver[1] + artifact_path = getAppDefinitionPath(project_dir, app_name) + if not artifact_path: + raise FileNotFoundError(f"No artifact definition file found for {app_name}") + app_def = Application.model_validate(openYaml(artifact_path)) + + auth_headers = app_def.registry.resolve_auth(cred_config) + + logger.info(f'Use template resolving new logic for {appver}') + tasks[template_type] = resolve_artifact_new_logic(app_def, app_version, template_dest, auth_headers) + + async def resolve_all(): + results = await asyncio.gather(*tasks.values()) + return dict(zip(tasks.keys(), results)) + + return asyncio.run(resolve_all()) diff --git a/scripts/build_env/env_template/set_template_version.py b/scripts/build_env/env_template/set_template_version.py index f5c3f37c0..c140954a1 100644 --- a/scripts/build_env/env_template/set_template_version.py +++ b/scripts/build_env/env_template/set_template_version.py @@ -4,12 +4,17 @@ from envgenehelper import beautifyYaml, writeYamlToFile, logger, getenv_with_error, getEnvDefinitionPath from envgenehelper import getEnvDefinition +from envgenehelper.models import TemplateVersionUpdateMode -def update_version(env_definition_dir, version_to_add): + +def update_version(env_definition_dir, version_to_add, update_mode: TemplateVersionUpdateMode): env_definition_path = getEnvDefinitionPath(env_definition_dir) logger.info(f"Started version update to {version_to_add} in {env_definition_path}.") data = getEnvDefinition(env_instances_dir) + if update_mode == TemplateVersionUpdateMode.TEMPORARY: + logger.info("Template update mode: TEMPORARY, Skip updating template artifact version in env_definition.yml") + return if ":" in version_to_add: if 'envTemplate' in data: if 'templateArtifact' in data['envTemplate']: @@ -39,4 +44,5 @@ def update_version(env_definition_dir, version_to_add): environment_name = getenv_with_error("ENVIRONMENT_NAME") env_instances_dir = Path(f"{base_dir}/environments/{cluster_name}/{environment_name}") version_to_add = getenv("ENV_TEMPLATE_VERSION") - update_version(env_instances_dir, version_to_add) + env_tmp_ver_update_mode = TemplateVersionUpdateMode(getenv("ENV_TEMPLATE_VERSION_UPDATE_MODE")) + update_version(env_instances_dir, version_to_add, env_tmp_ver_update_mode) diff --git a/scripts/build_env/filter_namespaces.py b/scripts/build_env/filter_namespaces.py index 19786738a..22a4a36e9 100644 --- a/scripts/build_env/filter_namespaces.py +++ b/scripts/build_env/filter_namespaces.py @@ -1,7 +1,9 @@ +from os import getenv + from pathlib import Path import shutil -from envgenehelper.business_helper import NamespaceFile, get_bgd_object, get_namespaces, get_namespaces_path, getenv_and_log, getenv_with_error +from envgenehelper.business_helper import NamespaceFile, get_bgd_object, get_namespaces, getenv_with_error from envgenehelper import logger def filter_namespaces(namespaces: list[str], filter: str, bgd_object: dict) -> list[str]: @@ -29,7 +31,10 @@ def filter_namespaces(namespaces: list[str], filter: str, bgd_object: dict) -> l return filtered_namespaces def apply_ns_build_filter(): - filter = getenv_and_log('NS_BUILD_FILTER', default='') + filter = getenv('NS_BUILD_FILTER') + if not filter: + logger.info('NS_BUILD_FILTER is empty, skipping filtering') + return logger.info(f"Filtering namespaces with NS_BUILD_FILTER: {filter}") base_dir = getenv_with_error("CI_PROJECT_DIR") diff --git a/scripts/build_env/main.py b/scripts/build_env/main.py index 8205d9441..fb12545a0 100644 --- a/scripts/build_env/main.py +++ b/scripts/build_env/main.py @@ -17,7 +17,7 @@ ENV_SPECIFIC_RESOURCE_PROFILE_SCHEMA = "schemas/resource-profile.schema.json" -def prepare_folders_for_rendering(env_name, cluster_name, source_env_dir, templates_dir, render_dir, +def prepare_folders_for_rendering(env_name, cluster_name, source_env_dir, templates_dirs, render_dir, render_parameters_dir, render_profiles_dir, output_dir): # clearing folders delete_dir(render_dir) @@ -28,8 +28,14 @@ def prepare_folders_for_rendering(env_name, cluster_name, source_env_dir, templa # clearing instances dir cleanup_resulting_dir(Path(output_dir) / cluster_name / env_name) # copying parameters from templates and instances - check_dir_exist_and_create(f'{render_parameters_dir}/from_template') - copy_path(f'{templates_dir}/parameters', f'{render_parameters_dir}/from_template') + for template_type, template_path in templates_dirs.items(): + if not (template_path and check_dir_exists(f'{template_path}/parameters')): + continue + if template_type == NamespaceRole.COMMON: + param_dir_name = 'from_template' + else: + param_dir_name = f'from_{template_type}_template' + copy_path(f'{template_path}/parameters', f'{render_parameters_dir}/{param_dir_name}') cluster_path = getDirName(source_env_dir) instances_dir = getDirName(cluster_path) check_dir_exist_and_create(f'{render_parameters_dir}/from_instance') @@ -37,7 +43,7 @@ def prepare_folders_for_rendering(env_name, cluster_name, source_env_dir, templa copy_path(f'{cluster_path}/parameters', render_parameters_dir) copy_path(f'{source_env_dir}/{INVENTORY_DIR_NAME}/parameters', f'{render_parameters_dir}/from_instance') # copying all template resource profiles - copy_path(f'{templates_dir}/resource_profiles', render_profiles_dir) + copy_path(f'{templates_dirs[NamespaceRole.COMMON]}/resource_profiles', render_profiles_dir) return render_env_dir @@ -95,7 +101,7 @@ def handle_template_override(render_dir): deleteFile(file) -def build_environment(env_name, cluster_name, templates_dir, source_env_dir, all_instances_dir, output_dir, work_dir): +def build_environment(env_name, cluster_name, templates_dirs, source_env_dir, all_instances_dir, output_dir, work_dir): # defining folders that will be used during generation base_dir = getenv_with_error('CI_PROJECT_DIR') render_dir = f"{base_dir}/tmp/render" @@ -108,7 +114,7 @@ def build_environment(env_name, cluster_name, templates_dir, source_env_dir, all shutil.copytree(get_namespaces_path(), os.path.join(work_dir,'build_env','tmp','initial_namespaces_content','Namespaces'), dirs_exist_ok=True) # preparing folders for generation - render_env_dir = prepare_folders_for_rendering(env_name, cluster_name, source_env_dir, templates_dir, render_dir, + render_env_dir = prepare_folders_for_rendering(env_name, cluster_name, source_env_dir, templates_dirs, render_dir, render_parameters_dir, render_profiles_dir, output_dir) pre_process_env_before_rendering(render_env_dir, source_env_dir, all_instances_dir) # get deployer parameters @@ -160,7 +166,8 @@ def build_environment(env_name, cluster_name, templates_dir, source_env_dir, all envvars["env"] = env_name # Keep as string for file paths envvars["current_env"] = current_env # Object for Jinja2 templates that need current_env.environmentName envvars["cluster_name"] = cluster_name - envvars["templates_dir"] = templates_dir + envvars["templates_dirs"] = templates_dirs + envvars["templates_dir"] = templates_dirs.get(NamespaceRole.COMMON, '') envvars["env_instances_dir"] = getAbsPath(render_env_dir) envvars["render_dir"] = getAbsPath(render_dir) envvars["render_parameters_dir"] = getAbsPath(render_parameters_dir) @@ -174,7 +181,7 @@ def build_environment(env_name, cluster_name, templates_dir, source_env_dir, all env_specific_resource_profile_map = get_env_specific_resource_profiles(source_env_dir, all_instances_dir, ENV_SPECIFIC_RESOURCE_PROFILE_SCHEMA) build_env(env_name, source_env_dir, render_parameters_dir, render_dir, render_profiles_dir, - env_specific_resource_profile_map, all_instances_dir, render_context) + env_specific_resource_profile_map, all_instances_dir, render_context, templates_dirs) resulting_dir = post_process_env_after_rendering(env_name, render_env_dir, source_env_dir, all_instances_dir, output_dir) @@ -263,22 +270,23 @@ def validate_parameter_files(param_files): return errors -def render_environment(env_name, cluster_name, templates_dir, all_instances_dir, output_dir, work_dir): +def render_environment(env_name, cluster_name, templates_dirs, all_instances_dir, output_dir, work_dir): logger.info(f'env: {env_name}') logger.info(f'cluster_name: {cluster_name}') - logger.info(f'templates_dir: {templates_dir}') + logger.info(f'templates_dirs: {templates_dirs}') logger.info(f'instances_dir: {all_instances_dir}') logger.info(f'output_dir: {output_dir}') logger.info(f'work_dir: {work_dir}') check_environment_is_valid_or_fail(env_name, cluster_name, all_instances_dir, validate_env_definition_by_schema=True) - # searching for env directory in instances - validate_parameters(templates_dir, all_instances_dir, cluster_name, env_name) + for _, template_dir in templates_dirs.items(): + if template_dir: + validate_parameters(template_dir, all_instances_dir, cluster_name, env_name) env_dir = get_env_instances_dir(env_name, cluster_name, all_instances_dir) logger.info(f"Environment {env_name} directory is {env_dir}") - resulting_env_dir = build_environment(env_name, cluster_name, templates_dir, env_dir, all_instances_dir, + resulting_env_dir = build_environment(env_name, cluster_name, templates_dirs, env_dir, all_instances_dir, output_dir, work_dir) create_credentials(resulting_env_dir, env_dir, all_instances_dir) apply_ns_build_filter() @@ -288,11 +296,11 @@ def render_environment(env_name, cluster_name, templates_dir, all_instances_dir, base_dir = getenv_with_error('CI_PROJECT_DIR') cluster = getenv_with_error("CLUSTER_NAME") environment = getenv_with_error("ENVIRONMENT_NAME") - g_templates_dir = f"{base_dir}/tmp/templates" + g_template_dirs = get_template_dirs() g_all_instances_dir = f"{base_dir}/environments" g_output_dir = f"{base_dir}/environments" g_work_dir = get_parent_dir_for_dir(g_all_instances_dir) - + decrypt_all_cred_files_for_env() - render_environment(environment, cluster, g_templates_dir, g_all_instances_dir, g_output_dir, g_work_dir) + render_environment(environment, cluster, g_template_dirs, g_all_instances_dir, g_output_dir, g_work_dir) encrypt_all_cred_files_for_env() diff --git a/scripts/build_env/handle_sd.py b/scripts/build_env/process_sd.py similarity index 95% rename from scripts/build_env/handle_sd.py rename to scripts/build_env/process_sd.py index 6602663d6..47c0d103a 100644 --- a/scripts/build_env/handle_sd.py +++ b/scripts/build_env/process_sd.py @@ -5,8 +5,9 @@ from os import path, getenv from pathlib import Path -import envgenehelper as helper import yaml + +import envgenehelper as helper from artifact_searcher import artifact from artifact_searcher.utils import models as artifact_models from envgenehelper.business_helper import getenv_and_log, getenv_with_error @@ -15,6 +16,7 @@ from envgenehelper.logger import logger from envgenehelper.plugin_engine import PluginEngine from envgenehelper.sd_merge_helper import basic_merge_multiple +from envgenehelper.collections_helper import split_multi_value_param class MergeType(Enum): @@ -188,13 +190,6 @@ def multiply_sds_to_single(sds_data, effective_merge_mode): def handle_sd(env, sd_source_type, sd_version, sd_data, sd_delta, sd_merge_mode): base_sd_path = Path(f'{env.env_path}/Inventory/solution-descriptor/') - handle_sd_skip_msg = "SD_SOURCE_TYPE is not specified, skipping SD file creation" - if not sd_source_type: - if not sd_version and not sd_data: - logger.info(handle_sd_skip_msg) - else: - logger.warning(handle_sd_skip_msg) - return sd_delta = calculate_sd_delta(sd_delta) effective_merge_mode = calculate_merge_mode(sd_merge_mode, sd_delta) @@ -277,7 +272,7 @@ def download_sds_with_version(env, base_sd_path, sd_version, effective_merge_mod logger.error("SD_SOURCE_TYPE is set to 'artifact', but SD_VERSION was not given in pipeline variables") exit(1) sd_version = sd_version.replace("\\n", "\n") - sd_entries = [line.strip() for line in sd_version.strip().splitlines() if line.strip()] + sd_entries = split_multi_value_param(sd_version) if not sd_entries: logger.error("No valid SD versions found in SD_VERSION") exit(1) @@ -306,12 +301,17 @@ def download_sd_by_appver(app_name: str, version: str, plugins: PluginEngine) -> # TODO: check if job would fail without plugins app_def = get_appdef_for_app(f"{app_name}:{version}", app_name, plugins) - artifact_info = asyncio.run(artifact.check_artifact_async(app_def, artifact.FileExtension.JSON, version)) + env_creds = helper.get_cred_config() + auth_headers = app_def.registry.resolve_auth(env_creds) + + artifact_info = asyncio.run( + artifact.check_artifact_async(app_def, artifact.FileExtension.JSON, version, + auth_headers=auth_headers)) if not artifact_info: raise ValueError( f'Solution descriptor content was not received for {app_name}:{version}') sd_url, _ = artifact_info - return artifact.download_json_content(sd_url) + return artifact.download_json_content(sd_url, auth_headers=auth_headers) def get_appdef_for_app(appver: str, app_name: str, plugins: PluginEngine) -> artifact_models.Application: @@ -322,7 +322,7 @@ def get_appdef_for_app(appver: str, app_name: str, plugins: PluginEngine) -> art app_def_path = identify_yaml_extension(f"{APP_DEFS_PATH}/{app_name}") app_dict = helper.openYaml(app_def_path) reg_def_path = identify_yaml_extension(f"{REG_DEFS_PATH}/{app_dict['registryName']}") - app_dict['registry'] = artifact_models.Registry.model_validate(helper.openYaml(reg_def_path)) + app_dict['registry'] = artifact_models.parse_registry(helper.openYaml(reg_def_path)) app_def = artifact_models.Application.model_validate(app_dict) return app_def diff --git a/scripts/build_env/render_config_env.py b/scripts/build_env/render_config_env.py index 6e8c02e79..8ea5383ef 100644 --- a/scripts/build_env/render_config_env.py +++ b/scripts/build_env/render_config_env.py @@ -5,7 +5,7 @@ from deepmerge import always_merger from envgenehelper import * -from envgenehelper.business_helper import get_bgd_object, get_namespaces +from envgenehelper.business_helper import get_bgd_object, get_namespaces, get_namespace_role, NamespaceRole from envgenehelper.validation import ensure_valid_fields, ensure_required_keys from jinja2 import Template, TemplateError from pydantic import BaseModel, Field @@ -14,6 +14,8 @@ from jinja.replace_ansible_stuff import replace_ansible_stuff, escaping_quotation SCHEMAS_DIR = Path(__file__).resolve().parents[2] / "schemas" +APPDEF_SCHEMA = str(SCHEMAS_DIR / "appdef.schema.json") +TD_SCHEMA = str(SCHEMAS_DIR / "template-descriptor.schema.json") yml = create_yaml_processor() @@ -22,13 +24,15 @@ class Context(BaseModel): env: Optional[str] = '' render_dir: Optional[str] = '' cloud_passport: OrderedDict = Field(default_factory=OrderedDict) - templates_dir: Optional[Path] = None + templates_dirs: dict = Field(default_factory=dict) output_dir: Optional[str] = '' cluster_name: Optional[str] = '' env_definition: OrderedDict = Field(default_factory=OrderedDict) current_env: OrderedDict = Field(default_factory=OrderedDict) current_env_dir: Optional[str] = '' current_env_template: OrderedDict = Field(default_factory=OrderedDict) + peer_env_template: OrderedDict = Field(default_factory=OrderedDict) + origin_env_template: OrderedDict = Field(default_factory=OrderedDict) tenant: Optional[str] = '' env_template: OrderedDict = Field(default_factory=OrderedDict) env_instances_dir: Optional[str] = '' @@ -47,6 +51,7 @@ class Context(BaseModel): render_profiles_dir: Optional[str] = '' start_time: datetime | None = Field(default=None, exclude=True) + end_time: datetime | None = Field(default=None, exclude=True) class Config: extra = "allow" @@ -67,7 +72,11 @@ def use(self): try: yield self finally: - logger.debug(f"Final state: {self.dict(exclude_none=True)}") + self.end_time = datetime.now() + logger.debug( + f"Exit context at {self.end_time}. Duration: {self.end_time - self.start_time}. " + f"Final state: {self.dict(exclude_none=True)}" + ) def as_dict(self, include_none: bool = False) -> dict: return self.model_dump(exclude_none=not include_none) @@ -104,16 +113,54 @@ def generate_config(self): logger.info(f"config = {config}") self.ctx.config = config - def set_env_template(self): - env_template_path_stem = f'{self.ctx.templates_dir}/env_templates/{self.ctx.current_env["env_template"]}' - env_template_path = next(iter(find_all_yaml_files_by_stem(env_template_path_stem)), None) + def find_env_template_in_dir(self, template_dir, env_template_name): + if not template_dir: + return None, [] + extensions = ("yml.j2", "yaml.j2", "yml", "yaml") + env_templates_dir = Path(f'{template_dir}/env_templates') + env_template_basename = env_templates_dir / env_template_name + suitable_files = find_files_by_basename(env_template_basename, extensions) + env_template_path = suitable_files[0] if suitable_files else None if not env_template_path: - raise ValueError(f'Template descriptor was not found in {env_template_path_stem}') - - env_template = openYaml(filePath=env_template_path, safe_load=True) + return None, suitable_files + + env_tmpl_final_path = env_template_path + if env_template_path.suffix.endswith("j2"): + logger.info(f"Template descriptor is {env_template_path}, rendering required") + env_tmpl_final_path = str(env_template_path).removesuffix(".j2") + self.render_from_file_to_file(env_template_path, env_tmpl_final_path) + + validate_yaml_by_scheme_or_fail(env_tmpl_final_path, TD_SCHEMA) + env_template = openYaml(filePath=env_tmpl_final_path, safe_load=True) + logger.info(f"Loaded env_template from {env_tmpl_final_path}") + return env_template, suitable_files + + def set_env_templates(self): + env_template_name = self.ctx.current_env["env_template"] + + env_templates_dir = Path(f'{self.ctx.templates_dirs.get(NamespaceRole.COMMON)}/env_templates') + env_template_basename = env_templates_dir / env_template_name + env_template, suitable_files = self.find_env_template_in_dir(self.ctx.templates_dirs.get(NamespaceRole.COMMON), env_template_name) + if not env_template: + all_files = [f for f in env_templates_dir.iterdir() if f.is_file()] + remains_files = list(set(all_files) - set(suitable_files)) + raise ValueError( + f"Template descriptor not found: {env_template_basename}." + f" Expected location in template repository: {env_template_basename}" + f" Allowed extensions: 'yml', 'yaml', 'yml.j2', 'yaml.j2'." + f" Found templates: {remains_files}") logger.info(f"env_template = {env_template}") self.ctx.current_env_template = env_template + if self.ctx.templates_dirs: + peer_template, _ = self.find_env_template_in_dir(self.ctx.templates_dirs.get(NamespaceRole.PEER), env_template_name) + if peer_template: + self.ctx.peer_env_template = peer_template + + origin_template, _ = self.find_env_template_in_dir(self.ctx.templates_dirs.get(NamespaceRole.ORIGIN), env_template_name) + if origin_template: + self.ctx.origin_env_template = origin_template + def setup_base_context(self, extra_env: dict): all_vars = dict(os.environ) self.ctx.update(extra_env) @@ -159,37 +206,61 @@ def validate_bgd(self): raise ValueError(f'Next namespaces were not found in available namespaces: {mismatch}') logger.info('Validation was successful') - def generate_ns_postfix(self, ns, ns_template_path, override_template_ns_name: str | None = None) -> str: - deploy_postfix = ns.get("deploy_postfix") - if deploy_postfix: - ns_template_name = deploy_postfix - else: - # get base name(deploy postfix) without extensions - ns_template_name = self.get_template_name(ns_template_path) - - ns_name = None - if override_template_ns_name: - ns_name = override_template_ns_name - elif ns_template_path: - rendered_ns = self.render_from_file_to_obj(ns_template_path) - ns_name = rendered_ns.get("name") - bgd = get_bgd_object(Path(f'{self.ctx.current_env_dir}')) - logger.debug(f'bgd object before comparing with ns: {bgd}') - if not bgd: - return ns_template_name - - origin_name = bgd["originNamespace"]["name"] - peer_name = bgd["peerNamespace"]["name"] - if ns_name == origin_name: - ns_template_name += "-origin" - elif ns_name == peer_name: - ns_template_name += "-peer" - logger.debug(f'After appending the ns name : {ns_template_name}') - return ns_template_name + def _get_bgd_suffix(self, ns_name: str | None) -> str: + if not ns_name: + return "" + bgd = get_bgd_object(Path(self.ctx.current_env_dir)) + role = get_namespace_role(ns_name, bgd) + return {NamespaceRole.ORIGIN: "-origin", NamespaceRole.PEER: "-peer"}.get(role, "") + + def _get_ns_name_for_bgd(self, ns: dict, ns_template_path: str) -> str | None: + override_name = self._fetch_template_override_name(ns) + if override_name: + return override_name + if ns_template_path: + return self.render_from_file_to_obj(ns_template_path).get("name") + return None + + def _get_template_dir_for_role(self, role: NamespaceRole) -> Path: + if role == NamespaceRole.ORIGIN and self.ctx.templates_dirs.get(NamespaceRole.ORIGIN): + return Path(self.ctx.templates_dirs[NamespaceRole.ORIGIN]) + if role == NamespaceRole.PEER and self.ctx.templates_dirs.get(NamespaceRole.PEER): + return Path(self.ctx.templates_dirs[NamespaceRole.PEER]) + return self.ctx.templates_dirs[NamespaceRole.COMMON] + + def _get_env_template_for_role(self, role: NamespaceRole) -> dict: + if role == NamespaceRole.ORIGIN and self.ctx.origin_env_template: + return self.ctx.origin_env_template + if role == NamespaceRole.PEER and self.ctx.peer_env_template: + return self.ctx.peer_env_template + return self.ctx.current_env_template + + def _resolve_template_path(self, template_path_expr: str, templates_dir: Path = None) -> str: + ctx = self.ctx.as_dict() + if templates_dir is not None: + ctx["templates_dir"] = str(templates_dir) + return Template(template_path_expr).render(ctx) + + def _find_ns_config_by_name(self, env_template: dict, ns_name: str, templates_dir: Path = None) -> dict | None: + for ns in env_template.get("namespaces", []): + override_name = self._fetch_template_override_name(ns) + if override_name == ns_name: + return ns + ns_template_path = self._resolve_template_path(ns["template_path"], templates_dir) + if ns_template_path: + rendered = self.render_from_file_to_obj(ns_template_path) + if rendered.get("name") == ns_name: + return ns + return None + + def generate_ns_postfix(self, ns: dict, ns_template_path: str) -> str: + base_name = ns.get("deploy_postfix") or self.get_template_name(ns_template_path) + ns_name = self._get_ns_name_for_bgd(ns, ns_template_path) + return base_name + self._get_bgd_suffix(ns_name) def generate_solution_structure(self): - sd_path_stem = f'{self.ctx.current_env_dir}/Inventory/solution-descriptor/sd' - sd_path = next(iter(find_all_yaml_files_by_stem(sd_path_stem)), None) + sd_basename = f'{self.ctx.current_env_dir}/Inventory/solution-descriptor/sd' + sd_path = next(iter(find_files_by_basename(sd_basename)), None) solution_structure = {} if sd_path: self.ctx.sd_file_path = str(sd_path) @@ -288,30 +359,43 @@ def generate_bgd_file(self): return self.render_from_file_to_file(Template(template).render(self.ctx.as_dict()), target_path) - def fetch_template_override_name(self, ns) -> str: - override_namespace_content = ns.get("template_override") - if override_namespace_content: - rendered = create_jinja_env().from_string(str(override_namespace_content)).render(self.ctx.as_dict()) - if rendered: - template_name = readYaml(rendered) - return template_name.get("name") - return "" + def _fetch_template_override_name(self, ns: dict) -> str: + template_override = ns.get("template_override") + if not template_override: + return "" + rendered = create_jinja_env().from_string(str(template_override)).render(self.ctx.as_dict()) + if not rendered: + return "" + return readYaml(rendered).get("name", "") - def generate_namespace_file(self): + def generate_namespace_files(self): context = self.ctx.as_dict() - namespaces = self.ctx.current_env_template["namespaces"] - for ns in namespaces: + bgd = get_bgd_object(Path(self.ctx.current_env_dir)) + + for ns in self.ctx.current_env_template["namespaces"]: ns_template_path = Template(ns["template_path"]).render(context) - override_template_ns_name = self.fetch_template_override_name(ns) - deploy_postfx = self.generate_ns_postfix(ns, ns_template_path, override_template_ns_name) - logger.info(f"Generate Namespace yaml for {deploy_postfx}") - current_env_dir = self.ctx.current_env_dir - ns_dir = f'{current_env_dir}/Namespaces/{deploy_postfx}' - namespace_file = f'{ns_dir}/namespace.yml' - self.render_from_file_to_file(ns_template_path, namespace_file) + postfix = self.generate_ns_postfix(ns, ns_template_path) + + ns_name = self._get_ns_name_for_bgd(ns, ns_template_path) + role = get_namespace_role(ns_name, bgd) if ns_name else NamespaceRole.COMMON - self.generate_override_template(ns.get("template_override"), Path(f'{ns_dir}/namespace.yml_override'), - deploy_postfx) + role_templates_dir = self._get_template_dir_for_role(role) + role_env_template = self._get_env_template_for_role(role) + + effective_ns = ns + effective_template_path = ns_template_path + + if role != NamespaceRole.COMMON and role_env_template is not self.ctx.current_env_template: + role_ns_config = self._find_ns_config_by_name(role_env_template, ns_name, role_templates_dir) + if role_ns_config: + effective_ns = role_ns_config + effective_template_path = self._resolve_template_path(role_ns_config["template_path"], role_templates_dir) + logger.info(f"Using {role.name} template for namespace {ns_name}") + + logger.info(f"Generate Namespace yaml for {postfix}") + ns_dir = Path(self.ctx.current_env_dir) / "Namespaces" / postfix + self.render_from_file_to_file(effective_template_path, str(ns_dir / "namespace.yml")) + self.generate_override_template(effective_ns.get("template_override"), ns_dir / "namespace.yml_override", postfix) def calculate_cloud_name(self) -> str: inv = self.ctx.env_definition["inventory"] @@ -382,35 +466,26 @@ def find_templates(self, path: str, patterns) -> list[Path]: def render_app_defs(self): for def_tmpl_path in self.ctx.appdef_templates: - app_def_str = openFileAsString(def_tmpl_path) - matches = re.findall( - r'^\s*(name|artifactId|groupId):\s*"([^"]+)"', - app_def_str, - flags=re.MULTILINE, - ) - appdef_meta = dict(matches) - ensure_valid_fields(appdef_meta, ["artifactId", "groupId", "name"]) - group_id = appdef_meta["groupId"] - artifact_id = appdef_meta["artifactId"] + app_def = self.render_from_file_to_obj(def_tmpl_path) + ensure_valid_fields(app_def, ["artifactId", "groupId", "name"]) + + app_name = app_def.get("name") + group_id = app_def["groupId"] + artifact_id = app_def["artifactId"] + self.ctx.update({ "app_lookup_key": f"{group_id}:{artifact_id}", "groupId": group_id, "artifactId": artifact_id, }) - app_def_trg_path = f"{self.ctx.current_env_dir}/AppDefs/{appdef_meta.get("name")}.yml" - self.render_from_file_to_file(def_tmpl_path, app_def_trg_path) + app_def_trg_path = f"{self.ctx.current_env_dir}/AppDefs/{app_name}.yml" + writeYamlToFile(app_def_trg_path, app_def) def render_reg_defs(self): for def_tmpl_path in self.ctx.regdef_templates: - reg_def_str = openFileAsString(def_tmpl_path) - matches = re.findall( - r'^\s*(name):\s*"([^"]+)"', - reg_def_str, - flags=re.MULTILINE, - ) - regdef_meta = dict(matches) - ensure_valid_fields(regdef_meta, ["name"]) - reg_def_trg_path = f"{self.ctx.current_env_dir}/RegDefs/{regdef_meta['name']}.yml" + reg_def = self.render_from_file_to_obj(def_tmpl_path) + ensure_valid_fields(reg_def, ["name"]) + reg_def_trg_path = f"{self.ctx.current_env_dir}/RegDefs/{reg_def.get('name')}.yml" self.render_from_file_to_file(def_tmpl_path, reg_def_trg_path) def set_appreg_def_overrides(self): @@ -482,7 +557,7 @@ def validate_appregdefs(self): logger.warning(f"No AppDef YAMLs found in {appdef_dir}") for file in appdef_files: logger.info(f"AppDef file: {file}") - validate_yaml_by_scheme_or_fail(file, str(SCHEMAS_DIR / "appdef.schema.json")) + validate_yaml_by_scheme_or_fail(file, APPDEF_SCHEMA) if os.path.exists(regdef_dir): regdef_files = findAllYamlsInDir(regdef_dir) @@ -490,7 +565,7 @@ def validate_appregdefs(self): logger.warning(f"No RegDef YAMLs found in {regdef_dir}") for file in regdef_files: logger.info(f"RegDef file: {file}") - validate_yaml_by_scheme_or_fail(file, str(SCHEMAS_DIR / "regdef.schema.json")) + validate_regdef_or_fail(file) def process_app_reg_defs(self, env_name: str, extra_env: dict): logger.info( @@ -499,7 +574,7 @@ def process_app_reg_defs(self, env_name: str, extra_env: dict): self.setup_base_context(extra_env) current_env_dir = self.ctx.current_env_dir - templates_dir = self.ctx.templates_dir + templates_dir = self.ctx.templates_dirs[NamespaceRole.COMMON] patterns = ["*.yaml.j2", "*.yml.j2", "*.j2", "*.yaml", "*.yml"] appdef_templates = self.find_templates(f"{templates_dir}/appdefs", patterns) regdef_templates = self.find_templates(f"{templates_dir}/regdefs", patterns) @@ -545,13 +620,13 @@ def render_config_env(self, env_name: str, extra_env: dict): current_env_dir = f'{self.ctx.render_dir}/{self.ctx.env}' self.ctx.current_env_dir = current_env_dir - self.set_env_template() + self.set_env_templates() self.generate_bgd_file() self.generate_solution_structure() self.generate_tenant_file() self.generate_cloud_file() - self.generate_namespace_file() + self.generate_namespace_files() self.generate_composite_structure() env_specific_schema = self.ctx.current_env_template.get("envSpecificSchema") diff --git a/scripts/build_env/resource_profiles.py b/scripts/build_env/resource_profiles.py index 77ab1ea41..b710ec9e1 100644 --- a/scripts/build_env/resource_profiles.py +++ b/scripts/build_env/resource_profiles.py @@ -1,4 +1,3 @@ -import copy from envgenehelper import * from render_config_env import EnvGenerator @@ -20,7 +19,8 @@ def get_env_specific_resource_profiles(env_dir, instances_dir, rp_schema): logger.debug(f"Searching for env specific resource profiles for template '{templateType}'") profileFileName = envSepcificResourceProfileNames[templateType] logger.debug(f"Searching for {profileFileName} for template type {templateType}") - resourceProfileFiles = findResourcesBottomTop(env_dir, instances_dir, f"/{profileFileName}.", f"{env_dir}/Profiles/") + resourceProfileFiles = findResourcesBottomTop(env_dir, instances_dir, f"/{profileFileName}.", + f"{env_dir}/Profiles/") if len(resourceProfileFiles) == 1: yamlPath = resourceProfileFiles[0] result[templateType] = yamlPath @@ -30,7 +30,8 @@ def get_env_specific_resource_profiles(env_dir, instances_dir, rp_schema): logger.error( f"Duplicate resource profile files with key '{profileFileName}' found in '{instances_dir}': \n\t" + ",\n\t".join( str(x) for x in resourceProfileFiles)) - raise ReferenceError( f"Duplicate resource profile files with key '{profileFileName}' found. See logs above.") + raise ReferenceError( + f"Duplicate resource profile files with key '{profileFileName}' found. See logs above.") else: raise ReferenceError(f"Resource profile file with key '{profileFileName}' not found in '{instances_dir}'") logger.info(f"Env specific resource profiles are: \n{dump_as_yaml_format(result)}") @@ -130,53 +131,58 @@ def validate_resource_profiles(needed_resource_profiles: dict[str, str], source_ return profiles_map -def processResourceProfiles(env_dir, resource_profiles_dir, profiles_schema, needed_resource_profiles_map, - env_specific_resource_profile_map, render_context: EnvGenerator, header_text=""): - logger.info(f"Needed profiles map: \n{dump_as_yaml_format(needed_resource_profiles_map)}") - render_context.generate_profiles(set(needed_resource_profiles_map.values())) +def collect_resource_profiles(result_profiles_dir, render_profiles_dir, profiles_schema, + required_resource_profiles_map, render_context: EnvGenerator): + logger.info(f"Required profiles map:\n{dump_as_yaml_format(required_resource_profiles_map)}") + render_context.generate_profiles(set(required_resource_profiles_map.values())) + all_profiles = getResourceProfilesFromDir(render_profiles_dir) | getResourceProfilesFromDir(result_profiles_dir) + logger.info(f"All existing resource profiles map is:\n{dump_as_yaml_format(all_profiles)}") + profiles_map = validate_resource_profiles(required_resource_profiles_map, all_profiles, profiles_schema) + return profiles_map + + +def override_by_env_specific_profiles(all_profiles, env_specific_resource_profile_map, render_context: EnvGenerator): + override_profile_map = {} render_context.generate_profiles(set(env_specific_resource_profile_map.values())) - # map for profiles from templates - templateProfilesMap = getResourceProfilesFromDir(resource_profiles_dir) - envRpDir = f"{env_dir}/Profiles" - environmentDirProfilesMap = getResourceProfilesFromDir(envRpDir) - # joining resource profiles with the result of Jinja generation - sourceProfilesMap = templateProfilesMap | environmentDirProfilesMap - logger.info(f"All resource profiles map is: \n{dump_as_yaml_format(sourceProfilesMap)}") - # check that all required resource profiles exists and are valid - profilesMap = validate_resource_profiles(needed_resource_profiles_map, sourceProfilesMap, profiles_schema) - # iterate through env specific resource profiles and perform override - for templateName, envSpecificProfileFile in env_specific_resource_profile_map.items(): - if templateName not in profilesMap: - logger.error( - f"No override profile for {templateName} found. Can't apply environment specific resource profile {envSpecificProfileFile}" - ) + for profile_key, env_specific_profile_path in env_specific_resource_profile_map.items(): + if profile_key not in all_profiles: raise ReferenceError( - f"Can't apply environment specific resource profile for namespace {templateName}. Please set override profile in templates first." + f"Environment specific profile '{env_specific_profile_path}' cannot be applied " + f"for profile key '{profile_key}', because no base template profile was found" ) - logger.info(f"Found template override profile for namespace '{templateName}' with environment specific profile {envSpecificProfileFile}") - templateProfileFilePath = profilesMap[templateName] - templateProfileYaml = openYaml(templateProfileFilePath) - envSpecificProfileYaml = openYaml(envSpecificProfileFile) + logger.info(f"Found template override profile for profile key '{profile_key}'" + f" with environment specific profile {env_specific_profile_path}") + template_profile_file_path = all_profiles[profile_key] + template_profile_yaml = openYaml(template_profile_file_path) + env_specific_profile_yaml = openYaml(env_specific_profile_path) combination_mode_key = "mergeEnvSpecificResourceProfiles" try: combination_mode = render_context.ctx.env_definition['inventory']['config'][combination_mode_key] except KeyError: - logger.info(f"inventory.config.{combination_mode_key} key not found in env_definition, default value is 'true'") + logger.info( + f"inventory.config.{combination_mode_key} key not found in env_definition, default value is 'true'") combination_mode = 'true' - common_msg = (f"profile overrides, because {combination_mode_key} is set to {combination_mode}") - # decide here whether to merge or replace + common_msg = f"profile overrides, because {combination_mode_key} is set to {combination_mode}" + if str(combination_mode).lower() == 'true': logger.info(f"Joining {common_msg}") - merge_resource_profiles(templateProfileYaml, envSpecificProfileYaml, extractNameFromFile(envSpecificProfileFile)) - writeYamlToFile(templateProfileFilePath, templateProfileYaml) + merge_resource_profiles(template_profile_yaml, env_specific_profile_yaml, + extractNameFromFile(env_specific_profile_path)) + writeYamlToFile(template_profile_file_path, template_profile_yaml) else: logger.info(f"Replacing {common_msg}") - profilesMap[templateName] = envSpecificProfileFile - - for profileKey, profileFilePath in profilesMap.items(): - logger.info(f"Copying '{profileKey}' to resulting directory '{envRpDir}'") - copy_path(profileFilePath, f"{envRpDir}/") - resultingProfilePath = f"{envRpDir}/{extractNameFromFile(profileFilePath)}.yml" - resultingProfilePath = identify_yaml_extension(resultingProfilePath) - beautifyYaml(resultingProfilePath, profiles_schema, header_text) + override_profile_map[profile_key] = env_specific_profile_path + return override_profile_map + + +def has_valid_profile_name(content: dict) -> bool: + profile = content.get("profile") + return isinstance(profile, dict) and bool(profile.get("name")) + + +def update_profile_name(file_path, profile_name): + data = openYaml(file_path, {}) + if has_valid_profile_name(data): + set_nested_yaml_attribute(data, "profile.name", profile_name) + writeYamlToFile(file_path, data) diff --git a/scripts/build_env/tests/app_reg_defs/test_appregdefs.py b/scripts/build_env/tests/app_reg_defs/test_appregdefs.py index bd98f04e4..e3df35a57 100644 --- a/scripts/build_env/tests/app_reg_defs/test_appregdefs.py +++ b/scripts/build_env/tests/app_reg_defs/test_appregdefs.py @@ -4,7 +4,10 @@ from pathlib import Path import pytest import yaml + from render_config_env import EnvGenerator +from envgenehelper.test_helpers import TestHelpers +from envgenehelper.business_helper import NamespaceRole class TestAppRegDefRendering: @@ -12,20 +15,20 @@ class TestAppRegDefRendering: @pytest.fixture(scope="class", autouse=True) def setup_test_environment(self, request): cls = request.cls - + test_base = Path(__file__).parents[4] / "test_data" / "test_app_reg_defs" cls.test_data_dir = test_base cls.cluster_name = "cluster-01" cls.env_name = "env-01" - + cls.output_dir = Path("/tmp/appregdefs") cls.output_dir.mkdir(parents=True, exist_ok=True) - + os.environ["CLUSTER_NAME"] = cls.cluster_name os.environ["ENVIRONMENT_NAME"] = cls.env_name - + yield - + os.environ.pop("CLUSTER_NAME", None) os.environ.pop("ENVIRONMENT_NAME", None) @@ -41,17 +44,19 @@ def _setup_render_dir(self) -> Path: def _get_render_context(self, test_number: str) -> dict: render_dir = self.output_dir / "render" / self.env_name - + test_case_dir = self._get_test_case_dir(test_number) - + env_dir = test_case_dir / "environments" / self.cluster_name / self.env_name templates_dir = test_case_dir / "templates" - + templates_dirs = {'common': str(templates_dir)} + return { "cluster_name": self.cluster_name, "output_dir": str(test_case_dir / "environments"), "current_env_dir": str(render_dir), "templates_dir": str(templates_dir), + "templates_dirs": templates_dirs, "cloud_passport_file_path": "", "env_instances_dir": str(env_dir) } @@ -61,50 +66,11 @@ def _verify_rendered_files(self, test_number: str, render_dir: Path): test_case_dir = self._get_test_case_dir(test_number) expected_appdefs = test_case_dir / "expected" / "appdefs" expected_regdefs = test_case_dir / "expected" / "regdefs" - - if expected_appdefs.exists(): - for expected_file in expected_appdefs.glob("*.y*ml"): - base_name = expected_file.stem - rendered_file = None - for ext in ['.yml', '.yaml']: - candidate = render_dir / "AppDefs" / f"{base_name}{ext}" - if candidate.exists(): - rendered_file = candidate - break - - assert rendered_file and rendered_file.exists(), \ - f"AppDef file {expected_file.name} should be rendered (checked {base_name}.yml and {base_name}.yaml)" - - with open(expected_file, encoding="utf-8") as f: - expected_content = yaml.safe_load(f) - with open(rendered_file, encoding="utf-8") as f: - rendered_content = yaml.safe_load(f) - - assert rendered_content == expected_content, \ - f"AppDef {expected_file.name} content mismatch.\nExpected: {expected_content}\nGot: {rendered_content}" - - if expected_regdefs.exists(): - for expected_file in expected_regdefs.glob("*.y*ml"): - base_name = expected_file.stem - rendered_file = None - for ext in ['.yml', '.yaml']: - candidate = render_dir / "RegDefs" / f"{base_name}{ext}" - if candidate.exists(): - rendered_file = candidate - break - - assert rendered_file and rendered_file.exists(), \ - f"RegDef file {expected_file.name} should be rendered (checked {base_name}.yml and {base_name}.yaml)" - - with open(expected_file) as f: - expected_content = yaml.safe_load(f) - with open(rendered_file) as f: - rendered_content = yaml.safe_load(f) - - assert rendered_content == expected_content, \ - f"RegDef {expected_file.name} content mismatch.\nExpected: {expected_content}\nGot: {rendered_content}" - - POSITIVE_CASES = [ + + TestHelpers.assert_dirs_content(expected_appdefs, render_dir / "AppDefs") + TestHelpers.assert_dirs_content(expected_regdefs, render_dir / "RegDefs") + + POSITIVE_CASES = [ "TC-001-001", "TC-001-002", "TC-001-003", @@ -114,29 +80,29 @@ def _verify_rendered_files(self, test_number: str, render_dir: Path): "TC-001-008", ] - @pytest.mark.parametrize("test_number", POSITIVE_CASES) + @pytest.mark.parametrize("test_number", POSITIVE_CASES) def test_positive_basic_appdef_rendering(self, test_number): self._setup_render_dir() - + render_context = EnvGenerator() context_vars = self._get_render_context(test_number) render_context.process_app_reg_defs(self.env_name, context_vars) - + render_dir = Path(context_vars["current_env_dir"]) self._verify_rendered_files(test_number, render_dir) - + NEGATIVE_CASES = { "TC-001-010": TemplateSyntaxError, "TC-001-011": ValueError, "TC-001-012": ValueError, } - + @pytest.mark.parametrize("test_number,expected_exception", NEGATIVE_CASES.items()) def test_negative_appregdef_rendering(self, test_number, expected_exception): self._setup_render_dir() - + render_context = EnvGenerator() context_vars = self._get_render_context(test_number) with pytest.raises(expected_exception): render_context.process_app_reg_defs(self.env_name, context_vars) - \ No newline at end of file + diff --git a/scripts/build_env/tests/env-build/test_paramset_sorting.py b/scripts/build_env/tests/env-build/test_paramset_sorting.py new file mode 100644 index 000000000..e88b7d102 --- /dev/null +++ b/scripts/build_env/tests/env-build/test_paramset_sorting.py @@ -0,0 +1,59 @@ +from envgenehelper.business_helper import is_from_template_dir +from build_env import sort_paramsets_with_same_name + +class TestSortParamsetsWithSameName: + + def test_all_three_levels(self): + entries = [ + {"filePath": "/tmp/render/parameters/from_instance/test.yml", "envSpecific": True}, + {"filePath": "/tmp/render/parameters/test.yml", "envSpecific": False}, + {"filePath": "/tmp/render/parameters/from_template/test.yml", "envSpecific": False}, + ] + sorted_entries = sort_paramsets_with_same_name(entries) + assert "from_template" in sorted_entries[0]["filePath"] + assert "from_instance" not in sorted_entries[1]["filePath"] + assert "from_instance" in sorted_entries[2]["filePath"] + + def test_template_and_instance(self): + entries = [ + {"filePath": "/tmp/render/parameters/from_instance/test.yml", "envSpecific": True}, + {"filePath": "/tmp/render/parameters/from_template/test.yml", "envSpecific": False}, + ] + sorted_entries = sort_paramsets_with_same_name(entries) + assert "from_template" in sorted_entries[0]["filePath"] + assert "from_instance" in sorted_entries[1]["filePath"] + + def test_origin_peer_templates(self): + entries = [ + {"filePath": "/tmp/render/parameters/from_instance/test.yml", "envSpecific": True}, + {"filePath": "/tmp/render/parameters/from_template/test.yml", "envSpecific": False}, + {"filePath": "/tmp/render/parameters/from_peer_template/test.yml", "envSpecific": False}, + {"filePath": "/tmp/render/parameters/from_origin_template/test.yml", "envSpecific": False}, + ] + sorted_entries = sort_paramsets_with_same_name(entries) + assert "from_origin_template" in sorted_entries[0]["filePath"] + assert "from_peer_template" in sorted_entries[1]["filePath"] + assert "from_template" in sorted_entries[2]["filePath"] + assert "from_instance" in sorted_entries[3]["filePath"] + + def test_multiple_files_sorted_alphabetically(self): + entries = [ + {"filePath": "/tmp/render/parameters/from_template/z_params.yml", "envSpecific": False}, + {"filePath": "/tmp/render/parameters/from_template/a_params.yml", "envSpecific": False}, + {"filePath": "/tmp/render/parameters/from_template/m_params.yml", "envSpecific": False}, + ] + sorted_entries = sort_paramsets_with_same_name(entries) + paths = [e["filePath"] for e in sorted_entries] + assert paths == sorted(paths) + + def test_real_world_dcl_e2e(self): + entries = [ + {"filePath": "/tmp/render/parameters/from_instance/DCL_E2E_parameters.yaml", "envSpecific": True}, + {"filePath": "/tmp/render/parameters/from_template/e2e/dcl.yaml", "envSpecific": False}, + ] + sorted_entries = sort_paramsets_with_same_name(entries) + assert "from_template" in sorted_entries[0]["filePath"] + assert "from_instance" in sorted_entries[1]["filePath"] + + def test_empty_list(self): + assert len(sort_paramsets_with_same_name([])) == 0 diff --git a/scripts/build_env/tests/env-build/test_render_envs.py b/scripts/build_env/tests/env-build/test_render_envs.py index a01f2ac6c..6f165b78a 100644 --- a/scripts/build_env/tests/env-build/test_render_envs.py +++ b/scripts/build_env/tests/env-build/test_render_envs.py @@ -2,23 +2,25 @@ import pytest from envgenehelper import * +from envgenehelper.business_helper import NamespaceRole -from main import render_environment, cleanup_resulting_dir +from main import render_environment from envgenehelper.test_helpers import TestHelpers from tests.base_test import BaseTest test_data = [ # (cluster_name, environment_name, template) - ("cluster-01", "env-01", "composite-prod"), - ("cluster-01", "env-02", "composite-dev"), - ("cluster-01", "env-03", "composite-dev"), - ("cluster-01", "env-04", "simple"), - ("cluster01", "env01", "test-01"), - ("cluster01", "env03", "test-template-1"), - ("cluster01", "env04", "test-template-2"), - ("bgd-cluster", "bgd-env", "bgd"), - ("cluster03", "rpo-replacement-mode", "simple"), + ("cluster-01", "env-01", "composite-prod", {}), + ("cluster-01", "env-02", "composite-dev", {}), + ("cluster-01", "env-03", "composite-dev", {}), + ("cluster-01", "env-04", "simple", {}), + ("cluster01", "env01", "test-01", {}), + ("cluster01", "env03", "test-template-1", {}), + ("cluster01", "env04", "test-template-2", {}), + ("bgd-cluster", "bgd-env", "bgd", {}), + ("bgd-cluster", "bgd-ns-artifacts-env", "bgd-ns-artifacts", {NamespaceRole.PEER: "test_data/test_templates_peer", NamespaceRole.ORIGIN: "test_data/test_templates_origin"}), + ("cluster03", "rpo-replacement-mode", "simple", {}), ] @@ -27,18 +29,23 @@ class TestEnvBuild(BaseTest): def change_test_dir(self, monkeypatch): monkeypatch.chdir(self.base_dir) - @pytest.mark.parametrize("cluster_name, env_name, version", test_data) - def test_render_envs(self, cluster_name, env_name, version): - g_templates_dir = str((self.test_data_dir / "test_templates").resolve()) + @pytest.mark.parametrize("cluster_name, env_name, version, extra_templates", test_data) + def test_render_envs(self, cluster_name, env_name, version, extra_templates): + g_templates_dirs = { + NamespaceRole.COMMON: str((self.test_data_dir / "test_templates").resolve()) + } + g_templates_dirs.update(extra_templates) + g_inventory_dir = str((self.test_data_dir / "test_environments").resolve()) g_output_dir = str((self.base_dir / "/tmp/test_environments").resolve()) os.environ['CI_COMMIT_REF_NAME'] = "branch_name" environ['FULL_ENV_NAME'] = cluster_name + '/' + env_name - render_environment(env_name, cluster_name, g_templates_dir, g_inventory_dir, g_output_dir, self.test_data_dir) + render_environment(env_name, cluster_name, g_templates_dirs, g_inventory_dir, g_output_dir, self.test_data_dir) source_dir = f"{g_inventory_dir}/{cluster_name}/{env_name}" generated_dir = f"{g_output_dir}/{cluster_name}/{env_name}" files_to_compare = get_all_files_in_dir(source_dir) logger.info(dump_as_yaml_format(files_to_compare)) TestHelpers.assert_dirs_content(source_dir, generated_dir, True, False) + diff --git a/scripts/build_env/tests/env-template/test_env_template.py b/scripts/build_env/tests/env-template/test_env_template.py index 1d0256351..14748dd67 100644 --- a/scripts/build_env/tests/env-template/test_env_template.py +++ b/scripts/build_env/tests/env-template/test_env_template.py @@ -1,4 +1,4 @@ -from os import environ +from os import environ, getenv from pathlib import Path import pytest @@ -43,6 +43,12 @@ f"{ARTIFACT_ZIP_ID}-{ZIP_VERSION}.zip" ) +SNAPSHOT_ZIP_URL = ( + f"{SNAPSHOT_BASE}/{PROJECT_GROUP_PATH}/" + f"{ARTIFACT_ZIP_ID}/{ZIP_VERSION}/" + f"{ARTIFACT_ZIP_ID}-{ZIP_VERSION}.zip" +) + TMPL_ZIP_URL = ( f"{TMPL_SNAPSHOT_BASE}/{PROJECT_GROUP_PATH}/" f"{ARTIFACT_ZIP_ID}/{ZIP_VERSION}/" @@ -94,9 +100,19 @@ def mock_aio_response(): }] } +dd_json_without_mvn_repo = { + "configurations": [{ + "artifacts": [{ + "id": f"{PROJECT_GROUP_ID}:{ARTIFACT_ZIP_ID}:{ZIP_VERSION}", + "type": "zip", + "classifier": "" + }] + }] +} def set_env(name: str): environ["ENVIRONMENT_NAME"] = name + environ["FULL_ENV_NAME"] = f"{getenv("CLUSTER_NAME")}/{getenv("ENVIRONMENT_NAME")}" def mock_metadata(aio_mock, url=METADATA_URL, repeat=1): @@ -115,6 +131,8 @@ def mock_dd_exists(aio_mock=None, exists=True): def mock_dd_response(): responses.add(responses.GET, DD_URL, json=dd_json, status=200) +def mock_dd_response_without_mvn_repo(): + responses.add(responses.GET, DD_URL, json=dd_json_without_mvn_repo, status=200) def mock_zip(url): responses.add( @@ -147,6 +165,7 @@ def init_env(self): environ.pop("CI_PROJECT_DIR", None) environ.pop("CLUSTER_NAME", None) environ.pop("ENVIRONMENT_NAME", None) + environ.pop("FULL_ENV_NAME", None) @responses.activate def test_new_logic_with_dd(self, mock_aio_response): @@ -163,6 +182,21 @@ def test_new_logic_with_dd(self, mock_aio_response): assert responses.calls[0].request.url == DD_URL assert responses.calls[1].request.url == STAGING_ZIP_URL + @responses.activate + def test_new_logic_with_dd_without_mvn_repo(self, mock_aio_response): + set_env("env-01") + + mock_metadata(mock_aio_response) + mock_dd_exists(mock_aio_response, exists=True) + mock_dd_response_without_mvn_repo() + mock_zip(SNAPSHOT_ZIP_URL) + + process_env_template() + + assert len(responses.calls) == 3 + assert responses.calls[0].request.url == DD_URL + assert responses.calls[1].request.url == SNAPSHOT_ZIP_URL + @responses.activate def test_new_logic_with_zip(self, mock_aio_response): set_env("env-01") diff --git a/scripts/build_env/tests/sd/test_handle_sd_artifact.py b/scripts/build_env/tests/sd/test_process_sd_artifact.py similarity index 95% rename from scripts/build_env/tests/sd/test_handle_sd_artifact.py rename to scripts/build_env/tests/sd/test_process_sd_artifact.py index 089402fd5..071308c21 100644 --- a/scripts/build_env/tests/sd/test_handle_sd_artifact.py +++ b/scripts/build_env/tests/sd/test_process_sd_artifact.py @@ -10,7 +10,7 @@ os.environ['CLUSTER_NAME'] = "temporary" os.environ['CI_PROJECT_DIR'] = "temporary" -from handle_sd import handle_sd +from process_sd import handle_sd from envgenehelper import * from envgenehelper.env_helper import Environment @@ -38,7 +38,7 @@ @pytest.mark.parametrize("test_case_name", TEST_CASES_POSITIVE) -@patch("handle_sd.download_sd_by_appver") +@patch("process_sd.download_sd_by_appver") def test_sd_positive(mock_download_sd, test_case_name): env = Environment(str(Path(OUTPUT_DIR, test_case_name)), "cluster-01", "env-01") do_prerequisites(SD, TEST_SD_DIR, OUTPUT_DIR, test_case_name, env, test_suits_map) @@ -60,7 +60,7 @@ def test_sd_positive(mock_download_sd, test_case_name): @pytest.mark.parametrize("test_case_name,expected_exception", [(k, v) for k, v in TEST_CASES_NEGATIVE.items()]) -@patch("handle_sd.download_sd_by_appver") +@patch("process_sd.download_sd_by_appver") def test_sd_negative(mock_download_sd, test_case_name, expected_exception): env = Environment(str(Path(OUTPUT_DIR, test_case_name)), "cluster-01", "env-01") do_prerequisites(SD, TEST_SD_DIR, OUTPUT_DIR, test_case_name, env, test_suits_map) diff --git a/scripts/build_env/tests/sd/test_handle_sd_local.py b/scripts/build_env/tests/sd/test_process_sd_local.py similarity index 98% rename from scripts/build_env/tests/sd/test_handle_sd_local.py rename to scripts/build_env/tests/sd/test_process_sd_local.py index de9edadae..d6a4a8082 100644 --- a/scripts/build_env/tests/sd/test_handle_sd_local.py +++ b/scripts/build_env/tests/sd/test_process_sd_local.py @@ -9,7 +9,7 @@ os.environ['CLUSTER_NAME'] = "temporary" os.environ['CI_PROJECT_DIR'] = "temporary" -from handle_sd import handle_sd +from process_sd import handle_sd from envgenehelper import * from envgenehelper.env_helper import Environment diff --git a/scripts/utils/entrypoint.sh b/scripts/utils/entrypoint.sh new file mode 100755 index 000000000..05efa517f --- /dev/null +++ b/scripts/utils/entrypoint.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +merge_java_opts() { + base="$1" + override="$2" + + result="$base" + allowed_override="" + + for opt in $override; do + case "$opt" in + -Xms*) + result=$(echo "$result" | sed -E 's/-Xms[^ ]+//g') + allowed_override="$opt${allowed_override:+ $allowed_override}" + ;; + -Xmx*) + result=$(echo "$result" | sed -E 's/-Xmx[^ ]+//g') + allowed_override="$opt${allowed_override:+ $allowed_override}" + ;; + -Djava.util.concurrent.ForkJoinPool.common.parallelism=*) + result=$(echo "$result" | sed -E 's|-Djava.util.concurrent.ForkJoinPool.common.parallelism=[^ ]+||g') + allowed_override="$opt${allowed_override:+ $allowed_override}" + ;; + esac + done + echo "$result${allowed_override:+ $allowed_override}" +} + +if [ -n "${CALCULATOR_CLI_JAVA_OPTIONS:-}" ]; then + JAVA_OPTIONS=$(merge_java_opts "$JAVA_OPTIONS" "$CALCULATOR_CLI_JAVA_OPTIONS" | tr -s '[:space:]') +fi +export JAVA_OPTIONS +exec /deployments/run-java.sh "$@" \ No newline at end of file diff --git a/build_envgene/scripts/handle_certs.sh b/scripts/utils/handle_certs.sh similarity index 86% rename from build_envgene/scripts/handle_certs.sh rename to scripts/utils/handle_certs.sh index 781bc5f3a..3332cb241 100755 --- a/build_envgene/scripts/handle_certs.sh +++ b/scripts/utils/handle_certs.sh @@ -15,7 +15,7 @@ fi if [ ! -d "$certs_dir" ] || ! find "$certs_dir" -mindepth 1 -print -quit >/dev/null 2>&1; then if [ -f "$default_cert" ]; then # shellcheck disable=SC1091 - . /module/scripts/update_ca_cert.sh "$default_cert" + . /module/scripts/utils/update_ca_cert.sh "$default_cert" else log "No certificates found and default certificate does not exist: $default_cert" fi @@ -23,6 +23,6 @@ else # Iterate files safely (handles spaces/newlines) while IFS= read -r -d '' cert; do # shellcheck disable=SC1091 - . /module/scripts/update_ca_cert.sh "$cert" + . /module/scripts/utils/update_ca_cert.sh "$cert" done < <(find "$certs_dir" -mindepth 1 -maxdepth 1 -type f -print0) fi \ No newline at end of file diff --git a/scripts/utils/pipeline_parameters.py b/scripts/utils/pipeline_parameters.py index e943febf5..05b21a3f6 100644 --- a/scripts/utils/pipeline_parameters.py +++ b/scripts/utils/pipeline_parameters.py @@ -1,62 +1,102 @@ import json +import uuid +import copy from os import getenv + from envgenehelper import logger from envgenehelper.plugin_engine import PluginEngine +from envgenehelper.models import TemplateVersionUpdateMode def get_pipeline_parameters() -> dict: return { 'ENV_NAMES': getenv("ENV_NAMES", ""), - 'ENV_BUILD': getenv("ENV_BUILDER") == "true", - 'GET_PASSPORT': getenv("GET_PASSPORT") == "true", - 'GENERATE_EFFECTIVE_SET': getenv("GENERATE_EFFECTIVE_SET", "false") == "true", + 'ENV_BUILD': getenv("ENV_BUILDER", "false").lower() == "true", + 'GET_PASSPORT': getenv("GET_PASSPORT", "false").lower() == "true", + 'GENERATE_EFFECTIVE_SET': getenv("GENERATE_EFFECTIVE_SET", "false").lower() == "true", 'ENV_TEMPLATE_VERSION': getenv("ENV_TEMPLATE_VERSION", ""), - 'ENV_TEMPLATE_TEST': getenv("ENV_TEMPLATE_TEST") == "true", - 'IS_TEMPLATE_TEST': getenv("ENV_TEMPLATE_TEST") == "true", + 'ENV_TEMPLATE_TEST': getenv("ENV_TEMPLATE_TEST", "false").lower() == "true", + 'IS_TEMPLATE_TEST': getenv("ENV_TEMPLATE_TEST", "false").lower() == "true", 'CI_COMMIT_REF_NAME': getenv("CI_COMMIT_REF_NAME", ""), 'JSON_SCHEMAS_DIR': getenv("JSON_SCHEMAS_DIR", "/module/schemas"), - "SD_SOURCE_TYPE": getenv("SD_SOURCE_TYPE"), + "SD_SOURCE_TYPE": getenv("SD_SOURCE_TYPE", "artifact"), "SD_VERSION": getenv("SD_VERSION"), "SD_DATA": getenv("SD_DATA"), "SD_DELTA": getenv("SD_DELTA"), "SD_REPO_MERGE_MODE": getenv("SD_REPO_MERGE_MODE"), - "ENV_INVENTORY_INIT": getenv("ENV_INVENTORY_INIT"), + "ENV_INVENTORY_INIT": getenv("ENV_INVENTORY_INIT", "false").lower() == "true", "ENV_SPECIFIC_PARAMS": getenv("ENV_SPECIFIC_PARAMS"), "ENV_TEMPLATE_NAME": getenv("ENV_TEMPLATE_NAME"), 'CRED_ROTATION_PAYLOAD': getenv("CRED_ROTATION_PAYLOAD", ""), - 'CRED_ROTATION_FORCE': getenv("CRED_ROTATION_FORCE", ""), + 'CRED_ROTATION_FORCE': getenv("CRED_ROTATION_FORCE", "false"), 'NS_BUILD_FILTER': getenv("NS_BUILD_FILTER", ""), - 'GITLAB_RUNNER_TAG_NAME' : getenv("GITLAB_RUNNER_TAG_NAME", ""), - 'RUNNER_SCRIPT_TIMEOUT' : getenv("RUNNER_SCRIPT_TIMEOUT") or "10m", - 'DEPLOYMENT_SESSION_ID': getenv("DEPLOYMENT_SESSION_ID", ""), - 'ENVGENE_LOG_LEVEL': getenv("ENVGENE_LOG_LEVEL"), - "BG_STATE": getenv("BG_STATE", None), - "BG_MANAGE": getenv("BG_MANAGE", None) == "true", - "ENV_INVENTORY_CONTENT": getenv("ENV_INVENTORY_CONTENT") + 'GITLAB_RUNNER_TAG_NAME': getenv("GITLAB_RUNNER_TAG_NAME", ""), + 'RUNNER_SCRIPT_TIMEOUT': getenv("RUNNER_SCRIPT_TIMEOUT", "10m"), + 'DEPLOYMENT_SESSION_ID': getenv("DEPLOYMENT_SESSION_ID", str(uuid.uuid4())), + 'ENVGENE_LOG_LEVEL': getenv("ENVGENE_LOG_LEVEL", "INFO"), + 'CALCULATOR_CLI_JAVA_OPTIONS' : getenv("CALCULATOR_CLI_JAVA_OPTIONS", ""), + "BG_STATE": getenv("BG_STATE"), + "BG_MANAGE": getenv("BG_MANAGE", "false").lower() == "true", + "APP_DEFS_PATH": getenv("APP_DEFS_PATH"), + "REG_DEFS_PATH": getenv("REG_DEFS_PATH"), + "APP_REG_DEFS_JOB": getenv("APP_REG_DEFS_JOB"), + "EFFECTIVE_SET_CONFIG" : getenv("EFFECTIVE_SET_CONFIG"), + "ENV_INVENTORY_CONTENT": getenv("ENV_INVENTORY_CONTENT"), + "CUSTOM_PARAMS" : getenv("CUSTOM_PARAMS"), + "ENV_TEMPLATE_VERSION_UPDATE_MODE": getenv( + "ENV_TEMPLATE_VERSION_UPDATE_MODE", TemplateVersionUpdateMode.PERSISTENT.value), } + +def get_sensitive_param_names() -> list: + return [ + "CRED_ROTATION_PAYLOAD", + "ENV_INVENTORY_CONTENT", + ] + class PipelineParametersHandler: def __init__(self, **kwargs): - plugins_dir='/module/scripts/pipegene_plugins/pipe_parameters' + plugins_dir = '/module/scripts/pipegene_plugins/pipe_parameters' self.params = get_pipeline_parameters() + self.sensitive_params = get_sensitive_param_names() pipe_param_plugin = PluginEngine(plugins_dir=plugins_dir) + if pipe_param_plugin.modules: - pipe_param_plugin.run(pipeline_params=self.params) + pipe_param_plugin.run(pipeline_params=self.params) + + for k, v in self.params.items(): + try: + parsed = json.loads(v) + self.params[k] = json.dumps(parsed, separators=(",", ":")) + + except (TypeError, ValueError): + pass + + def hide_secrets(self, data): + if isinstance(data, dict): + for k, v in data.items(): + if k.lower() in {"username", "password", "secret"}: + data[k] = "***" + else: + self.hide_secrets(v) + elif isinstance(data, list): + for item in data: + self.hide_secrets(item) def log_pipeline_params(self): params_str = "Input parameters are: " - params = self.params.copy() + params = copy.deepcopy(self.params) if params.get("CRED_ROTATION_PAYLOAD"): params["CRED_ROTATION_PAYLOAD"] = "***" + env_inventory_content = params.get("ENV_INVENTORY_CONTENT") + if env_inventory_content: + parsed = json.loads(env_inventory_content) + self.hide_secrets(parsed) + params["ENV_INVENTORY_CONTENT"] = json.dumps(parsed, separators=(",", ":")) + for k, v in params.items(): - try: - parsed = json.loads(v) - params[k] = json.dumps(parsed, separators=(",", ":")) - except (TypeError, ValueError): - pass - - params_str += f"\n{k.upper()}: {params[k]}" + params_str += f"\n{k.upper()}: {v}" logger.info(params_str) diff --git a/scripts/utils/update_ca_cert.sh b/scripts/utils/update_ca_cert.sh new file mode 100755 index 000000000..a1a76a25a --- /dev/null +++ b/scripts/utils/update_ca_cert.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +CA_FILE="$1" +# Default log level to INFO if not set; normalize to uppercase for comparison +ENVGENE_LOG_LEVEL="${ENVGENE_LOG_LEVEL:-INFO}" +ENVGENE_LOG_LEVEL="$(printf '%s' "${ENVGENE_LOG_LEVEL}" | tr '[:lower:]' '[:upper:]')" + +function getLinuxDisto { + if [[ -f /etc/os-release ]]; then + # freedesktop.org and systemd + . /etc/os-release + DIST=$NAME + elif type lsb_release >/dev/null 2>&1; then + # linuxbase.org + DIST=$(lsb_release -si) + elif [[ -f /etc/lsb-release ]]; then + # For some versions of Debian/Ubuntu without lsb_release command + . /etc/lsb-release + DIST=$DISTRIB_ID + elif [[ -f /etc/debian_version ]]; then + # Older Debian/Ubuntu/etc. + DIST=Debian + else + # Fall back to uname, e.g. "Linux ", also works for BSD, etc. + DIST=$(uname -s) + fi + # convert to lowercase + DIST="$(tr '[:upper:]' '[:lower:]' <<< "$DIST")" +} + +function debugPrintCertsFromFile { + local file="$1" + local label="$2" + # Exit early unless debug is enabled + [[ "${ENVGENE_LOG_LEVEL}" != "DEBUG" ]] && return + echo "[DEBUG] === ${label} ===" + if [[ ! -e "$file" ]]; then + echo "[DEBUG] File does not exist: $file" + return + fi + local cert_num=0 + local block="" + while IFS= read -r line || [[ -n "$line" ]]; do + line="${line%$'\r'}" + if [[ "$line" == "-----BEGIN CERTIFICATE-----" ]]; then + block="$line" + continue + fi + if [[ -n "$block" ]]; then + block+=$'\n'"$line" + if [[ "$line" == "-----END CERTIFICATE-----" ]]; then + cert_num=$((cert_num + 1)) + echo "[DEBUG] --- Certificate #${cert_num} in ${file} ---" + printf "%s\n" "$block" | openssl x509 -noout -subject -issuer -dates 2>/dev/null || echo "[DEBUG] (openssl could not decode this block)" + block="" + fi + fi + done < "$file" + if [[ $cert_num -eq 0 ]]; then + echo "[DEBUG] No PEM certificate blocks found in ${file}" + else + echo "[DEBUG] Total: ${cert_num} certificate(s)" + fi + echo "[DEBUG] === End ${label} ===" +} + +function updateCertificates { + if [[ -e "${CA_FILE}" && ! -z "${CA_FILE}" ]]; then + getLinuxDisto + echo "Linux Distribution identified as: $DIST" + + # Debug: print certificates in source file BEFORE import + debugPrintCertsFromFile "${CA_FILE}" "Certificates in source file BEFORE import (${CA_FILE})" + + # Derive destination filename from source so multiple CAs do not overwrite each other; use .crt for compatibility + LOCAL_NAME="$(basename "${CA_FILE}" | sed 's/\.[^.]*$//').crt" + if [[ "${DIST}" == *"debian"* || "${DIST}" == *"ubuntu"* ]]; then + cp "${CA_FILE}" "/usr/local/share/ca-certificates/${LOCAL_NAME}" + update-ca-certificates --fresh + echo "certs from ${CA_FILE} added to trusted root" + export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt #https://ubuntu.com/server/docs/install-a-root-ca-certificate-in-the-trust-store + elif [[ "${DIST}" == *"centos"* ]]; then + cp "${CA_FILE}" "/etc/pki/ca-trust/source/anchors/${LOCAL_NAME}" + update-ca-trust + echo "certs from ${CA_FILE} added to trusted root" + export REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt #https://techjourney.net/update-add-ca-certificates-bundle-in-redhat-centos/ + elif [[ "${DIST}" == *"alpine"* ]]; then + cp "${CA_FILE}" "/usr/local/share/ca-certificates/${LOCAL_NAME}" + update-ca-certificates + echo "certs from ${CA_FILE} added to trusted root" + export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt + elif [[ "${DIST}" == *"red hat"* ]]; then + mkdir -p /etc/pki/ca-trust/source/anchors + cp "${CA_FILE}" "/etc/pki/ca-trust/source/anchors/${LOCAL_NAME}" + update-ca-trust + echo "certs from ${CA_FILE} added to trusted root" + export REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt #https://www.redhat.com/en/blog/configure-ca-trust-list + fi + + # Debug: print certificates AFTER import (from the installed location / bundle) + if [[ "${DIST}" == *"debian"* || "${DIST}" == *"ubuntu"* ]]; then + debugPrintCertsFromFile "/usr/local/share/ca-certificates/${LOCAL_NAME}" "Certificates AFTER import (installed file /usr/local/share/ca-certificates/${LOCAL_NAME})" + elif [[ "${DIST}" == *"centos"* || "${DIST}" == *"red hat"* ]]; then + debugPrintCertsFromFile "/etc/pki/ca-trust/source/anchors/${LOCAL_NAME}" "Certificates AFTER import (installed file /etc/pki/ca-trust/source/anchors/${LOCAL_NAME})" + elif [[ "${DIST}" == *"alpine"* ]]; then + debugPrintCertsFromFile "/usr/local/share/ca-certificates/${LOCAL_NAME}" "Certificates AFTER import (installed file /usr/local/share/ca-certificates/${LOCAL_NAME})" + fi + [[ "${ENVGENE_LOG_LEVEL}" == "DEBUG" ]] && echo "[DEBUG] Certificate import completed successfully for ${CA_FILE}" + else + echo "CA file ${CA_FILE} not found or empty" + exit 1 + fi + echo "export REQUESTS_CA_BUNDLE=${REQUESTS_CA_BUNDLE}" >> ~/.bashrc +} + +updateCertificates diff --git a/src/assembly/assembly.xml b/src/assembly/assembly.xml deleted file mode 100644 index 6ab61e251..000000000 --- a/src/assembly/assembly.xml +++ /dev/null @@ -1,18 +0,0 @@ - - templates - - zip - - false - - - build_envgene_templates - / - - **/* - - - - diff --git a/test_data/pipegene_ci_instance/GAV_coordinates.yaml b/test_data/pipegene_ci_instance/GAV_coordinates.yaml new file mode 100644 index 000000000..8ba62720e --- /dev/null +++ b/test_data/pipegene_ci_instance/GAV_coordinates.yaml @@ -0,0 +1,6 @@ +--- +# Template-test fixture for pipegene (see pipeline_helper.get_gav_coordinates_from_build). +artifact: + group_id: org.qubership.test + artifact_id: app_def + version: new-version diff --git a/test_data/pipegene_ci_instance/configuration/integration.yml b/test_data/pipegene_ci_instance/configuration/integration.yml new file mode 100644 index 000000000..c82a4034d --- /dev/null +++ b/test_data/pipegene_ci_instance/configuration/integration.yml @@ -0,0 +1,6 @@ +cp_discovery: + gitlab: + project: "value" + branch: master + token: somevalue.secret +self_token: somevalue.secret diff --git a/test_data/pipegene_ci_instance/environments/cluster-01 b/test_data/pipegene_ci_instance/environments/cluster-01 new file mode 120000 index 000000000..ec4338dde --- /dev/null +++ b/test_data/pipegene_ci_instance/environments/cluster-01 @@ -0,0 +1 @@ +../../test_environments/cluster-01 \ No newline at end of file diff --git a/test_data/pipegene_ci_instance/templates/env_templates/composite-full.yml b/test_data/pipegene_ci_instance/templates/env_templates/composite-full.yml new file mode 100644 index 000000000..1b64520ae --- /dev/null +++ b/test_data/pipegene_ci_instance/templates/env_templates/composite-full.yml @@ -0,0 +1,3 @@ +--- +# Placeholder for ENV_TEMPLATE_TEST pipeline generation tests. +placeholder: true diff --git a/test_data/test_app_reg_defs/TC-001-001/expected/appdefs/application-1.yml b/test_data/test_app_reg_defs/TC-001-001/expected/appdefs/application-1.yml new file mode 100644 index 000000000..056998b9f --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-001/expected/appdefs/application-1.yml @@ -0,0 +1,7 @@ +name: application-1 +registryName: registry-1 +artifactId: application-1 +groupId: org.qubership +supportParallelDeploy: true +deployParameters: {} +technicalConfigurationParameters: {} diff --git a/test_data/test_app_reg_defs/TC-001-001/templates/appdefs/application-1.yaml.j2 b/test_data/test_app_reg_defs/TC-001-001/templates/appdefs/application-1.yaml.j2 index b70639ff3..5d25e6652 100644 --- a/test_data/test_app_reg_defs/TC-001-001/templates/appdefs/application-1.yaml.j2 +++ b/test_data/test_app_reg_defs/TC-001-001/templates/appdefs/application-1.yaml.j2 @@ -1,7 +1,7 @@ -name: "application-1" -registryName: "{{ appdefs.overrides.registryName | default('registry-1') }}" -artifactId: "application-1" -groupId: "org.qubership" +name: application-1 +registryName: {{ appdefs.overrides.registryName | default('registry-1') }} +artifactId: application-1 +groupId: org.qubership supportParallelDeploy: true deployParameters: {} technicalConfigurationParameters: {} diff --git a/test_data/test_app_reg_defs/TC-001-002/expected/regdefs/registry-1.yaml b/test_data/test_app_reg_defs/TC-001-002/expected/regdefs/registry-1.yml similarity index 100% rename from test_data/test_app_reg_defs/TC-001-002/expected/regdefs/registry-1.yaml rename to test_data/test_app_reg_defs/TC-001-002/expected/regdefs/registry-1.yml diff --git a/test_data/test_app_reg_defs/TC-001-003/expected/appdefs/application-1.yaml b/test_data/test_app_reg_defs/TC-001-003/expected/appdefs/application-1.yml similarity index 100% rename from test_data/test_app_reg_defs/TC-001-003/expected/appdefs/application-1.yaml rename to test_data/test_app_reg_defs/TC-001-003/expected/appdefs/application-1.yml diff --git a/test_data/test_app_reg_defs/TC-001-004/expected/regdefs/registry-1.yaml b/test_data/test_app_reg_defs/TC-001-004/expected/regdefs/registry-1.yml similarity index 95% rename from test_data/test_app_reg_defs/TC-001-004/expected/regdefs/registry-1.yaml rename to test_data/test_app_reg_defs/TC-001-004/expected/regdefs/registry-1.yml index 4f4b1d908..d75d7b480 100644 --- a/test_data/test_app_reg_defs/TC-001-004/expected/regdefs/registry-1.yaml +++ b/test_data/test_app_reg_defs/TC-001-004/expected/regdefs/registry-1.yml @@ -16,4 +16,4 @@ dockerConfig: snapshotRepoName: "docker-snapshot" stagingRepoName: "docker-staging" releaseRepoName: "docker-release" - groupName: "docker-group" \ No newline at end of file + groupName: "docker-group" diff --git a/test_data/test_app_reg_defs/TC-001-005/expected/appdefs/application-1.yaml b/test_data/test_app_reg_defs/TC-001-005/expected/appdefs/application-1.yml similarity index 100% rename from test_data/test_app_reg_defs/TC-001-005/expected/appdefs/application-1.yaml rename to test_data/test_app_reg_defs/TC-001-005/expected/appdefs/application-1.yml diff --git a/test_data/test_app_reg_defs/TC-001-006/expected/appdefs/application-1.yaml b/test_data/test_app_reg_defs/TC-001-006/expected/appdefs/application-1.yml similarity index 100% rename from test_data/test_app_reg_defs/TC-001-006/expected/appdefs/application-1.yaml rename to test_data/test_app_reg_defs/TC-001-006/expected/appdefs/application-1.yml diff --git a/test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-1.yaml b/test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-1.yaml deleted file mode 100644 index cf96b3568..000000000 --- a/test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-1.yaml +++ /dev/null @@ -1,7 +0,0 @@ -name: "application-1" -registryName: "registry-1" -artifactId: "application-1" -groupId: "org.qubership" -supportParallelDeploy: true -deployParameters: {} -technicalConfigurationParameters: {} diff --git a/test_data/test_app_reg_defs/TC-001-001/expected/appdefs/application-1.yaml b/test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-1.yml similarity index 100% rename from test_data/test_app_reg_defs/TC-001-001/expected/appdefs/application-1.yaml rename to test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-1.yml diff --git a/test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-2.yml.j2 b/test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-2.yml similarity index 100% rename from test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-2.yml.j2 rename to test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-2.yml diff --git a/test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-3.yml b/test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-3.yml index 53606066f..5aa758575 100644 --- a/test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-3.yml +++ b/test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-3.yml @@ -4,4 +4,4 @@ artifactId: "application-3" groupId: "org.qubership" supportParallelDeploy: true deployParameters: {} -technicalConfigurationParameters: {} \ No newline at end of file +technicalConfigurationParameters: {} diff --git a/test_data/test_environments/bgd-cluster/bgd-env/cloud.yml b/test_data/test_environments/bgd-cluster/bgd-env/cloud.yml index 3936201d9..4e7e338bc 100644 --- a/test_data/test_environments/bgd-cluster/bgd-env/cloud.yml +++ b/test_data/test_environments/bgd-cluster/bgd-env/cloud.yml @@ -14,7 +14,7 @@ dbMode: "db" databases: [] maasConfig: credentialsId: "maas" - enable: true + enable: false maasUrl: "http://maas.None" maasInternalAddress: "http://maas.maas:8888" vaultConfig: @@ -23,12 +23,12 @@ vaultConfig: url: "" dbaasConfigs: - credentialsId: "dbaas" - enable: true + enable: false apiUrl: "http://dbaas.dbaas:8888" aggregatorUrl: "https://dbaas.None" consulConfig: tokenSecret: "consul-token" - enabled: true + enabled: false publicUrl: "https://consul.None" internalUrl: "http://consul.consul:8888" deployParameters: diff --git a/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/Credentials/credentials.yml b/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/Credentials/credentials.yml new file mode 100644 index 000000000..d66d4ef15 --- /dev/null +++ b/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/Credentials/credentials.yml @@ -0,0 +1,22 @@ +bgdomaincred: # bg domain bgd-ns-artifacts-env-bg-domain + type: "secret" + data: + secret: "envgeneNullValue" # FillMe +cloud-deploy-sa-token: # cloud passport: test-cloud-passport version: 1.5 + type: "secret" + data: + secret: "token-placeholder-123" +consul-token: # cloud test-solution-structure + type: "secret" + data: + secret: "envgeneNullValue" # FillMe +dbaas: # cloud test-solution-structure + type: "usernamePassword" + data: + username: "envgeneNullValue" # FillMe + password: "envgeneNullValue" # FillMe +maas: # cloud test-solution-structure + type: "usernamePassword" + data: + username: "envgeneNullValue" # FillMe + password: "envgeneNullValue" # FillMe diff --git a/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/Inventory/env_definition.yml b/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/Inventory/env_definition.yml new file mode 100644 index 000000000..2113c553e --- /dev/null +++ b/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/Inventory/env_definition.yml @@ -0,0 +1,20 @@ +inventory: + environmentName: "bgd-ns-artifacts-env" + clusterUrl: "test-val.com" + tenantName: "test-tenant" + deployer: "test-deployer" + cloudName: "test-solution-structure" + cloudPassport: "test-cloud-passport" +envTemplate: + name: "bgd-ns-artifacts" + bgNsArtifacts: + origin: "bgd-ns-artifacts:bgd-ns-artifacts" + peer: "bgd-ns-artifacts:bgd-ns-artifacts" + additionalTemplateVariables: {} + sharedTemplateVariables: [] + envSpecificParamsets: {} + envSpecificTechnicalParamsets: {} + sharedMasterCredentialFiles: [] + artifact: "bgd-ns-artifacts:bgd-ns-artifacts" +generatedVersions: + generateEnvironmentLatestVersion: "bgd-ns-artifacts:bgd-ns-artifacts" diff --git a/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/Namespaces/app-origin/namespace.yml b/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/Namespaces/app-origin/namespace.yml new file mode 100644 index 000000000..2d3a68533 --- /dev/null +++ b/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/Namespaces/app-origin/namespace.yml @@ -0,0 +1,22 @@ +# The contents of this file is generated from template artifact: bgd-ns-artifacts. +# Contents will be overwritten by next generation. +# Please modify this contents only for development purposes or as workaround. +name: "bgd-ns-artifacts-env-origin-app" +credentialsId: "" +isServerSideMerge: false +labels: + - "Instance-bgd-ns-artifacts-env" +cleanInstallApprovalRequired: false +mergeDeployParametersAndE2EParameters: false +deployParameters: + APP_NAMESPACE: "bgd-ns-artifacts-env-app" + ENVGENE_CONFIG_REF_NAME: "branch_name" + ENVGENE_CONFIG_TAG: "No Ref tag" + ROLE_SPECIFIC_PARAM: "ORIGIN" +e2eParameters: + APP_NAMESPACE: "bgd-ns-artifacts-env-app" +technicalConfigurationParameters: + APP_NAMESPACE: "bgd-ns-artifacts-env-app" +deployParameterSets: [] +e2eParameterSets: [] +technicalConfigurationParameterSets: [] diff --git a/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/Namespaces/app-peer/namespace.yml b/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/Namespaces/app-peer/namespace.yml new file mode 100644 index 000000000..5d4e25e4f --- /dev/null +++ b/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/Namespaces/app-peer/namespace.yml @@ -0,0 +1,23 @@ +# The contents of this file is generated from template artifact: bgd-ns-artifacts. +# Contents will be overwritten by next generation. +# Please modify this contents only for development purposes or as workaround. +name: "bgd-ns-artifacts-env-peer-app" +credentialsId: "" +isServerSideMerge: false +labels: + - "Instance-bgd-ns-artifacts-env" +cleanInstallApprovalRequired: false +mergeDeployParametersAndE2EParameters: false +deployParameters: + APP_NAMESPACE: "bgd-ns-artifacts-env-app" + ENVGENE_CONFIG_REF_NAME: "branch_name" + ENVGENE_CONFIG_TAG: "No Ref tag" + ROLE_SPECIFIC_PARAM: "PEER" + param-A: "value from peer template version of paramset" # paramset: paramset-A version: n/a source: template +e2eParameters: + APP_NAMESPACE: "bgd-ns-artifacts-env-app" +technicalConfigurationParameters: + APP_NAMESPACE: "bgd-ns-artifacts-env-app" +deployParameterSets: [] +e2eParameterSets: [] +technicalConfigurationParameterSets: [] diff --git a/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/Namespaces/bg-controller/namespace.yml b/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/Namespaces/bg-controller/namespace.yml new file mode 100644 index 000000000..b691aa2d4 --- /dev/null +++ b/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/Namespaces/bg-controller/namespace.yml @@ -0,0 +1,16 @@ +# The contents of this file is generated from template artifact: bgd-ns-artifacts. +# Contents will be overwritten by next generation. +# Please modify this contents only for development purposes or as workaround. +name: "bgd-ns-artifacts-env-bg-controller" +credentialsId: "" +isServerSideMerge: false +labels: + - "Instance-bgd-ns-artifacts-env" +cleanInstallApprovalRequired: false +mergeDeployParametersAndE2EParameters: false +deployParameters: {} +e2eParameters: {} +technicalConfigurationParameters: {} +deployParameterSets: [] +e2eParameterSets: [] +technicalConfigurationParameterSets: [] diff --git a/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/bg_domain.yml b/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/bg_domain.yml new file mode 100644 index 000000000..fb00a5539 --- /dev/null +++ b/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/bg_domain.yml @@ -0,0 +1,12 @@ +name: "bgd-ns-artifacts-env-bg-domain" +type: bgdomain +originNamespace: + name: "bgd-ns-artifacts-env-origin-app" + type: namespace +peerNamespace: + name: "bgd-ns-artifacts-env-peer-app" + type: namespace +controllerNamespace: + name: "bgd-ns-artifacts-env-bg-controller" + type: namespace + credentials: bgdomaincred diff --git a/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/cloud.yml b/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/cloud.yml new file mode 100644 index 000000000..980276436 --- /dev/null +++ b/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/cloud.yml @@ -0,0 +1,48 @@ +# The contents of this file is generated from template artifact: bgd-ns-artifacts. +# Contents will be overwritten by next generation. +# Please modify this contents only for development purposes or as workaround. +name: "test-solution-structure" +apiUrl: "tmp.cloud.com" # cloud passport: test-cloud-passport version: 1.5 +apiPort: "0000" # cloud passport: test-cloud-passport version: 1.5 +privateUrl: "test-host.managed.tmp.cloud" # cloud passport: test-cloud-passport version: 1.5 +publicUrl: "test-host.managed.tmp.cloud" # cloud passport: test-cloud-passport version: 1.5 +dashboardUrl: "https://dashboard.test-host.managed.tmp.cloud" # cloud passport: test-cloud-passport version: 1.5 +labels: [] +defaultCredentialsId: "cloud-deploy-sa-token" # cloud passport: test-cloud-passport version: 1.5 +protocol: "https" # cloud passport: test-cloud-passport version: 1.5 +dbMode: "db" +databases: [] +maasConfig: + credentialsId: "maas" + enable: false + maasUrl: "http://maas.None" + maasInternalAddress: "http://maas.maas:8888" +vaultConfig: + credentialsId: "" + enable: false + url: "" +dbaasConfigs: + - credentialsId: "dbaas" + enable: false + apiUrl: "http://dbaas.dbaas:8888" + aggregatorUrl: "https://dbaas.None" +consulConfig: + tokenSecret: "consul-token" + enabled: false + publicUrl: "https://consul.None" + internalUrl: "http://consul.consul:8888" +deployParameters: + CLOUD_DASHBOARD_URL: "https://dashboard.test-host.managed.tmp.cloud" # cloud passport: test-cloud-passport version: 1.5 + CMDB_URL: "https://test-host.managed.tmp.cloud" # cloud passport: test-cloud-passport version: 1.5 + GRAFANA_UI_URL: "https://test-host.managed.tmp.cloud" # cloud passport: test-cloud-passport version: 1.5 + GRAYLOG_UI_URL: "https://test-host.managed.tmp.cloud" # cloud passport: test-cloud-passport version: 1.5 + TRACING_UI_URL: "https://test-host.managed.tmp.cloud" # cloud passport: test-cloud-passport version: 1.5 + VALUE: "true" # cloud passport: test-cloud-passport version: 1.5 +e2eParameters: + CMDB_NAME: "test-deployer" + TEMPLATE_NAME: "bgd-ns-artifacts" +technicalConfigurationParameters: {} +deployParameterSets: [] +e2eParameterSets: [] +technicalConfigurationParameterSets: [] +productionMode: false # cloud passport: test-cloud-passport version: 1.5 diff --git a/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/tenant.yml b/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/tenant.yml new file mode 100644 index 000000000..93914e1fe --- /dev/null +++ b/test_data/test_environments/bgd-cluster/bgd-ns-artifacts-env/tenant.yml @@ -0,0 +1,17 @@ +# The contents of this file is generated from template artifact: bgd-ns-artifacts. +# Contents will be overwritten by next generation. +# Please modify this contents only for development purposes or as workaround. +name: "test-tenant" +registryName: "default" +description: "" +owners: "" +gitRepository: "" +defaultBranch: "" +credential: "" +labels: [] +globalE2EParameters: + pipelineDefaultRecipients: "" + recipientsStrategy: "merge" + mergeTenantsAndE2EParameters: false + environmentParameters: {} +deployParameters: {} diff --git a/test_data/test_environments/cluster-01/env-01/cloud.yml b/test_data/test_environments/cluster-01/env-01/cloud.yml index 482a75bfd..4fa2aec6b 100644 --- a/test_data/test_environments/cluster-01/env-01/cloud.yml +++ b/test_data/test_environments/cluster-01/env-01/cloud.yml @@ -40,7 +40,7 @@ deployParameters: CDN_STORAGE_USERNAME: "${STORAGE_USERNAME}" # cloud passport: cluster-01 version: 1.5 CLOUD_DASHBOARD_URL: "https://dashboard.cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 CMDB_URL: "https://cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 - CONSUL_ENABLED: "true" # cloud passport: cluster-01 version: 1.5 + CONSUL_ENABLED: true # cloud passport: cluster-01 version: 1.5 DEFAULT_TENANT_ADMIN_LOGIN: "admin" # cloud passport: cluster-01 version: 1.5 DEFAULT_TENANT_ADMIN_PASSWORD: "password" # cloud passport: cluster-01 version: 1.5 DEFAULT_TENANT_NAME: "tenant" # cloud passport: cluster-01 version: 1.5 diff --git a/test_data/test_environments/cluster-01/env-02/cloud.yml b/test_data/test_environments/cluster-01/env-02/cloud.yml index 9b99c3901..855b9e8bf 100644 --- a/test_data/test_environments/cluster-01/env-02/cloud.yml +++ b/test_data/test_environments/cluster-01/env-02/cloud.yml @@ -40,7 +40,7 @@ deployParameters: CDN_STORAGE_USERNAME: "${STORAGE_USERNAME}" # cloud passport: cluster-01 version: 1.5 CLOUD_DASHBOARD_URL: "https://dashboard.cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 CMDB_URL: "https://cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 - CONSUL_ENABLED: "true" # cloud passport: cluster-01 version: 1.5 + CONSUL_ENABLED: true # cloud passport: cluster-01 version: 1.5 DEFAULT_TENANT_ADMIN_LOGIN: "admin" # cloud passport: cluster-01 version: 1.5 DEFAULT_TENANT_ADMIN_PASSWORD: "password" # cloud passport: cluster-01 version: 1.5 DEFAULT_TENANT_NAME: "tenant" # cloud passport: cluster-01 version: 1.5 diff --git a/test_data/test_environments/cluster-01/env-03/cloud.yml b/test_data/test_environments/cluster-01/env-03/cloud.yml index 19b15aecf..bf4322d8a 100644 --- a/test_data/test_environments/cluster-01/env-03/cloud.yml +++ b/test_data/test_environments/cluster-01/env-03/cloud.yml @@ -40,7 +40,7 @@ deployParameters: CDN_STORAGE_USERNAME: "${STORAGE_USERNAME}" # cloud passport: cluster-01 version: 1.5 CLOUD_DASHBOARD_URL: "https://dashboard.cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 CMDB_URL: "https://cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 - CONSUL_ENABLED: "true" # cloud passport: cluster-01 version: 1.5 + CONSUL_ENABLED: true # cloud passport: cluster-01 version: 1.5 DEFAULT_TENANT_ADMIN_LOGIN: "admin" # cloud passport: cluster-01 version: 1.5 DEFAULT_TENANT_ADMIN_PASSWORD: "password" # cloud passport: cluster-01 version: 1.5 DEFAULT_TENANT_NAME: "tenant" # cloud passport: cluster-01 version: 1.5 diff --git a/test_data/test_environments/cluster-01/env-04/cloud.yml b/test_data/test_environments/cluster-01/env-04/cloud.yml index cbacd692f..59fc611db 100644 --- a/test_data/test_environments/cluster-01/env-04/cloud.yml +++ b/test_data/test_environments/cluster-01/env-04/cloud.yml @@ -40,7 +40,7 @@ deployParameters: CDN_STORAGE_USERNAME: "${STORAGE_USERNAME}" # cloud passport: cluster-01 version: 1.5 CLOUD_DASHBOARD_URL: "https://dashboard.cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 CMDB_URL: "https://cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 - CONSUL_ENABLED: "true" # cloud passport: cluster-01 version: 1.5 + CONSUL_ENABLED: true # cloud passport: cluster-01 version: 1.5 DEFAULT_TENANT_ADMIN_LOGIN: "admin" # cloud passport: cluster-01 version: 1.5 DEFAULT_TENANT_ADMIN_PASSWORD: "password" # cloud passport: cluster-01 version: 1.5 DEFAULT_TENANT_NAME: "tenant" # cloud passport: cluster-01 version: 1.5 diff --git a/test_data/test_environments/cluster-02/env-01/cloud.yml b/test_data/test_environments/cluster-02/env-01/cloud.yml index 482a75bfd..4fa2aec6b 100644 --- a/test_data/test_environments/cluster-02/env-01/cloud.yml +++ b/test_data/test_environments/cluster-02/env-01/cloud.yml @@ -40,7 +40,7 @@ deployParameters: CDN_STORAGE_USERNAME: "${STORAGE_USERNAME}" # cloud passport: cluster-01 version: 1.5 CLOUD_DASHBOARD_URL: "https://dashboard.cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 CMDB_URL: "https://cluster-01.qubership.org" # cloud passport: cluster-01 version: 1.5 - CONSUL_ENABLED: "true" # cloud passport: cluster-01 version: 1.5 + CONSUL_ENABLED: true # cloud passport: cluster-01 version: 1.5 DEFAULT_TENANT_ADMIN_LOGIN: "admin" # cloud passport: cluster-01 version: 1.5 DEFAULT_TENANT_ADMIN_PASSWORD: "password" # cloud passport: cluster-01 version: 1.5 DEFAULT_TENANT_NAME: "tenant" # cloud passport: cluster-01 version: 1.5 diff --git a/test_data/test_environments/cluster01/env01/cloud.yml b/test_data/test_environments/cluster01/env01/cloud.yml index 2ed02be7d..84d87c2ae 100644 --- a/test_data/test_environments/cluster01/env01/cloud.yml +++ b/test_data/test_environments/cluster01/env01/cloud.yml @@ -15,7 +15,7 @@ databases: [] mergeDeployParametersAndE2EParameters: false maasConfig: credentialsId: "maas" - enable: true + enable: false maasUrl: "http://maas.qubership.org" maasInternalAddress: "http://maas.maas:8888" vaultConfig: @@ -24,12 +24,12 @@ vaultConfig: url: "" dbaasConfigs: - credentialsId: "dbaas" - enable: true + enable: false apiUrl: "http://dbaas.dbaas:8888" aggregatorUrl: "https://dbaas.qubership.org" consulConfig: tokenSecret: "consul-token" - enabled: true + enabled: false publicUrl: "https://consul.qubership.org" internalUrl: "http://consul.consul:8888" deployParameters: diff --git a/test_data/test_environments/cluster01/env03/cloud.yml b/test_data/test_environments/cluster01/env03/cloud.yml index 40e534ba9..78d604201 100644 --- a/test_data/test_environments/cluster01/env03/cloud.yml +++ b/test_data/test_environments/cluster01/env03/cloud.yml @@ -14,7 +14,7 @@ dbMode: "db" databases: [] maasConfig: credentialsId: "maas" - enable: true + enable: false maasUrl: "http://maas." maasInternalAddress: "http://maas.maas:8888" vaultConfig: @@ -23,12 +23,12 @@ vaultConfig: url: "" dbaasConfigs: - credentialsId: "dbaas" - enable: true + enable: false apiUrl: "http://dbaas.dbaas:8888" aggregatorUrl: "https://dbaas." consulConfig: tokenSecret: "consul-token" - enabled: true + enabled: false publicUrl: "https://consul." internalUrl: "http://consul.consul:8888" deployParameters: diff --git a/test_data/test_environments/cluster01/env04/Inventory/env_definition.yml b/test_data/test_environments/cluster01/env04/Inventory/env_definition.yml index ee80de1bd..3c31100c5 100644 --- a/test_data/test_environments/cluster01/env04/Inventory/env_definition.yml +++ b/test_data/test_environments/cluster01/env04/Inventory/env_definition.yml @@ -6,7 +6,10 @@ inventory: cloudPassport: "" envTemplate: name: "test-template-2" - additionalTemplateVariables: {} + additionalTemplateVariables: + namespaces: + app: + enabled: true templateArtifact: registry: "artifactory" repository: "snapshotRepository" diff --git a/test_data/test_environments/cluster03/rpo-replacement-mode/Namespaces/billing/namespace.yml b/test_data/test_environments/cluster03/rpo-replacement-mode/Namespaces/billing/namespace.yml index 2cd8de8ee..707fa0f3c 100644 --- a/test_data/test_environments/cluster03/rpo-replacement-mode/Namespaces/billing/namespace.yml +++ b/test_data/test_environments/cluster03/rpo-replacement-mode/Namespaces/billing/namespace.yml @@ -10,8 +10,8 @@ labels: cleanInstallApprovalRequired: true mergeDeployParametersAndE2EParameters: false profile: - name: "dev_billing_override" baseline: "dev" + name: "medium_billing_override" deployParameters: ENVGENE_CONFIG_REF_NAME: "branch_name" ENVGENE_CONFIG_TAG: "No Ref tag" diff --git a/test_data/test_templates/env_templates/bgd-ns-artifacts.yaml b/test_data/test_templates/env_templates/bgd-ns-artifacts.yaml new file mode 100644 index 000000000..716cda109 --- /dev/null +++ b/test_data/test_templates/env_templates/bgd-ns-artifacts.yaml @@ -0,0 +1,14 @@ +--- +tenant: "{{ templates_dir }}/env_templates/bgd/tenant.yml.j2" +cloud: + template_path: "{{ templates_dir }}/env_templates/bgd/cloud.yml.j2" +namespaces: + - template_path: "{{ templates_dir }}/env_templates/bgd-ns-artifacts/Namespaces/app.yml.j2" + template_override: + name: "{{ current_env.name }}-peer-app" + - template_path: "{{ templates_dir }}/env_templates/bgd-ns-artifacts/Namespaces/app.yml.j2" + template_override: + name: "{{ current_env.name }}-origin-app" + - template_path: "{{ templates_dir }}/env_templates/bgd/Namespaces/bg-controller.yml.j2" +bg_domain: "{{ templates_dir }}/env_templates/bgd/bg_domain.yml.j2" +parametersets: [] diff --git a/test_data/test_templates/env_templates/bgd-ns-artifacts/Namespaces/app.yml.j2 b/test_data/test_templates/env_templates/bgd-ns-artifacts/Namespaces/app.yml.j2 new file mode 100644 index 000000000..9c1cf3a9e --- /dev/null +++ b/test_data/test_templates/env_templates/bgd-ns-artifacts/Namespaces/app.yml.j2 @@ -0,0 +1,20 @@ +--- +name: "{{current_env.name}}-bss" +labels: +- "Instance-{{current_env.name}}" +credentialsId: "" +isServerSideMerge: false +cleanInstallApprovalRequired: false +mergeDeployParametersAndE2EParameters: false +deployParameters: + ENVGENE_CONFIG_REF_NAME: "{{ lookup('ansible.builtin.env', 'CI_COMMIT_REF_NAME')| default('No Ref Name') }}" + ENVGENE_CONFIG_TAG: "{{ lookup('ansible.builtin.env', 'CI_COMMIT_TAG')| default('No Ref tag') }}" + APP_NAMESPACE: "{{current_env.name}}-app" + ROLE_SPECIFIC_PARAM: "COMMON" +e2eParameters: + APP_NAMESPACE: "{{current_env.name}}-app" +technicalConfigurationParameters: + APP_NAMESPACE: "{{current_env.name}}-app" +deployParameterSets: [] +e2eParameterSets: [] +technicalConfigurationParameterSets: [] diff --git a/test_data/test_templates/env_templates/test-template-2.yaml b/test_data/test_templates/env_templates/test-template-2.yaml index bda1f5ca0..e143c4da1 100644 --- a/test_data/test_templates/env_templates/test-template-2.yaml +++ b/test_data/test_templates/env_templates/test-template-2.yaml @@ -3,5 +3,5 @@ tenant: "{{ templates_dir }}/env_templates/test-template-2/tenant.yml.j2" cloud: template_path: "{{ templates_dir }}/env_templates/test-template-2/cloud.yml.j2" namespaces: - - template_path: "{{ templates_dir }}/env_templates/test-template-2/Namespaces/app.yml.j2" - deploy_postfix: "app" + - template_path: "{{ templates_dir }}/env_templates/test-template-2/Namespaces/app-2.yml.j2" + deploy_postfix: "app-2" diff --git a/test_data/test_templates/env_templates/test-template-2.yaml.j2 b/test_data/test_templates/env_templates/test-template-2.yaml.j2 new file mode 100644 index 000000000..494b9bb7a --- /dev/null +++ b/test_data/test_templates/env_templates/test-template-2.yaml.j2 @@ -0,0 +1,11 @@ +--- +tenant: {{ templates_dir }}/env_templates/test-template-2/tenant.yml.j2 +cloud: + template_path: {{ templates_dir }}/env_templates/test-template-2/cloud.yml.j2 +namespaces: +{% if current_env.additionalTemplateVariables.namespaces["app"].enabled | default(False, True) %} + - template_path: {{ templates_dir }}/env_templates/test-template-2/Namespaces/app.yml.j2 + deploy_postfix: "app" +{% else %} + [] +{% endif %} diff --git a/test_data/test_templates_origin/env_templates/bgd-ns-artifacts.yaml b/test_data/test_templates_origin/env_templates/bgd-ns-artifacts.yaml new file mode 100644 index 000000000..716cda109 --- /dev/null +++ b/test_data/test_templates_origin/env_templates/bgd-ns-artifacts.yaml @@ -0,0 +1,14 @@ +--- +tenant: "{{ templates_dir }}/env_templates/bgd/tenant.yml.j2" +cloud: + template_path: "{{ templates_dir }}/env_templates/bgd/cloud.yml.j2" +namespaces: + - template_path: "{{ templates_dir }}/env_templates/bgd-ns-artifacts/Namespaces/app.yml.j2" + template_override: + name: "{{ current_env.name }}-peer-app" + - template_path: "{{ templates_dir }}/env_templates/bgd-ns-artifacts/Namespaces/app.yml.j2" + template_override: + name: "{{ current_env.name }}-origin-app" + - template_path: "{{ templates_dir }}/env_templates/bgd/Namespaces/bg-controller.yml.j2" +bg_domain: "{{ templates_dir }}/env_templates/bgd/bg_domain.yml.j2" +parametersets: [] diff --git a/test_data/test_templates_origin/env_templates/bgd-ns-artifacts/Namespaces/app.yml.j2 b/test_data/test_templates_origin/env_templates/bgd-ns-artifacts/Namespaces/app.yml.j2 new file mode 100644 index 000000000..7e950d1b0 --- /dev/null +++ b/test_data/test_templates_origin/env_templates/bgd-ns-artifacts/Namespaces/app.yml.j2 @@ -0,0 +1,20 @@ +--- +name: "{{current_env.name}}-bss" +labels: +- "Instance-{{current_env.name}}" +credentialsId: "" +isServerSideMerge: false +cleanInstallApprovalRequired: false +mergeDeployParametersAndE2EParameters: false +deployParameters: + ENVGENE_CONFIG_REF_NAME: "{{ lookup('ansible.builtin.env', 'CI_COMMIT_REF_NAME')| default('No Ref Name') }}" + ENVGENE_CONFIG_TAG: "{{ lookup('ansible.builtin.env', 'CI_COMMIT_TAG')| default('No Ref tag') }}" + APP_NAMESPACE: "{{current_env.name}}-app" + ROLE_SPECIFIC_PARAM: "ORIGIN" +e2eParameters: + APP_NAMESPACE: "{{current_env.name}}-app" +technicalConfigurationParameters: + APP_NAMESPACE: "{{current_env.name}}-app" +deployParameterSets: [] +e2eParameterSets: [] +technicalConfigurationParameterSets: [] diff --git a/test_data/test_templates_peer/env_templates/bgd-ns-artifacts.yaml b/test_data/test_templates_peer/env_templates/bgd-ns-artifacts.yaml new file mode 100644 index 000000000..716cda109 --- /dev/null +++ b/test_data/test_templates_peer/env_templates/bgd-ns-artifacts.yaml @@ -0,0 +1,14 @@ +--- +tenant: "{{ templates_dir }}/env_templates/bgd/tenant.yml.j2" +cloud: + template_path: "{{ templates_dir }}/env_templates/bgd/cloud.yml.j2" +namespaces: + - template_path: "{{ templates_dir }}/env_templates/bgd-ns-artifacts/Namespaces/app.yml.j2" + template_override: + name: "{{ current_env.name }}-peer-app" + - template_path: "{{ templates_dir }}/env_templates/bgd-ns-artifacts/Namespaces/app.yml.j2" + template_override: + name: "{{ current_env.name }}-origin-app" + - template_path: "{{ templates_dir }}/env_templates/bgd/Namespaces/bg-controller.yml.j2" +bg_domain: "{{ templates_dir }}/env_templates/bgd/bg_domain.yml.j2" +parametersets: [] diff --git a/test_data/test_templates_peer/env_templates/bgd-ns-artifacts/Namespaces/app.yml.j2 b/test_data/test_templates_peer/env_templates/bgd-ns-artifacts/Namespaces/app.yml.j2 new file mode 100644 index 000000000..1ac9e9c61 --- /dev/null +++ b/test_data/test_templates_peer/env_templates/bgd-ns-artifacts/Namespaces/app.yml.j2 @@ -0,0 +1,21 @@ +--- +name: "{{current_env.name}}-bss" +labels: +- "Instance-{{current_env.name}}" +credentialsId: "" +isServerSideMerge: false +cleanInstallApprovalRequired: false +mergeDeployParametersAndE2EParameters: false +deployParameters: + ENVGENE_CONFIG_REF_NAME: "{{ lookup('ansible.builtin.env', 'CI_COMMIT_REF_NAME')| default('No Ref Name') }}" + ENVGENE_CONFIG_TAG: "{{ lookup('ansible.builtin.env', 'CI_COMMIT_TAG')| default('No Ref tag') }}" + APP_NAMESPACE: "{{current_env.name}}-app" + ROLE_SPECIFIC_PARAM: "PEER" +e2eParameters: + APP_NAMESPACE: "{{current_env.name}}-app" +technicalConfigurationParameters: + APP_NAMESPACE: "{{current_env.name}}-app" +deployParameterSets: +- paramset-A +e2eParameterSets: [] +technicalConfigurationParameterSets: [] diff --git a/test_data/test_templates_peer/parameters/paramset-A.yaml b/test_data/test_templates_peer/parameters/paramset-A.yaml new file mode 100644 index 000000000..00b398d3b --- /dev/null +++ b/test_data/test_templates_peer/parameters/paramset-A.yaml @@ -0,0 +1,4 @@ +--- +name: "paramset-A" +parameters: + param-A: "value from peer template version of paramset"