diff --git a/isvctl/configs/aws/eks-layered.yaml b/isvctl/configs/aws/eks-layered.yaml new file mode 100644 index 00000000..acf6b981 --- /dev/null +++ b/isvctl/configs/aws/eks-layered.yaml @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# AWS EKS - Layered Provider Configuration +# +# Supplies commands (Terraform stubs) and context overrides for the +# kaas template. Use with the template for composable validation: +# +# isvctl test run \ +# -f isvctl/configs/templates/kaas.yaml \ +# -f isvctl/configs/aws/eks-layered.yaml +# +# For a self-contained config (no template needed), use eks.yaml instead. +# +# Environment Variables: +# TF_AUTO_APPROVE — Skip Terraform approval prompts (default: false) +# TF_VAR_* — Terraform variables (region, instance types, etc.) +# AWS_SKIP_TEARDOWN — Preserve resources after test (default: false) +# NGC_API_KEY — NGC API key for NIM workloads +# AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY — AWS credentials + +version: "1.0" + +# ============================================================================= +# Provider-specific commands (Terraform stubs) +# ============================================================================= +commands: + kubernetes: + phases: ["setup", "test", "teardown"] + steps: + - name: provision_cluster + phase: setup + command: "../stubs/aws/eks/setup.sh" + timeout: 1800 + env: + TF_AUTO_APPROVE: "true" + TF_VAR_region: "us-west-2" + TF_VAR_gpu_node_instance_types: '["g5.xlarge"]' + TF_VAR_gpu_node_desired_size: "1" + # Override with your IP CIDR for security (e.g., '["203.0.113.0/24"]') + # TF_VAR_cluster_endpoint_public_access_cidrs: '["0.0.0.0/0"]' + + - name: teardown_cluster + phase: teardown + command: "../stubs/aws/eks/teardown.sh" + timeout: 1800 + env: + TF_AUTO_APPROVE: "true" + +# ============================================================================= +# Context overrides for template defaults +# ============================================================================= +# Single g5.xlarge GPU node = 1 GPU total (template defaults to 3) +context: + total_gpus: "1" + +# ============================================================================= +# Validation overrides (optional) +# ============================================================================= +# The template provides all checks with sensible defaults. Override only +# what differs for this provider. Use {__merge__: true} to merge into +# the template's check list instead of replacing it. +tests: + description: "AWS EKS GPU cluster validation" diff --git a/isvctl/configs/carbide/README.md b/isvctl/configs/carbide/README.md new file mode 100644 index 00000000..445e2e01 --- /dev/null +++ b/isvctl/configs/carbide/README.md @@ -0,0 +1,88 @@ +# Carbide Provider + +Provider implementation for [NVIDIA Carbide](https://github.com/fabiendupont/cloud-provider-nvidia-carbide) bare-metal infrastructure management. Uses `carbidecli` to interact with the Carbide API. + +## Template Coverage + +| Template | Carbide Config | Status | +|----------|---------------|--------| +| `control-plane` | `control-plane.yaml` | SSH key + VPC lifecycle | +| `network` | `network.yaml` | VPC, subnet, prefix, NSG | +| `image-registry` | `image-registry.yaml` | OperatingSystem + InstanceType CRUD | +| `bm` | `bm.yaml` | Instance launch/describe/reboot/teardown | +| `iam` | `iam.yaml` | Token validation, scope coverage, write access | +| `vm` | — | Not applicable (VMs are a platform concern, e.g., KubeVirt) | +| `kaas` | — | Not applicable (K8s installation is a platform concern) | + +## Usage + +```bash +# IAM: token validation + scope coverage (run first) +isvctl test run \ + -f isvctl/configs/templates/iam.yaml \ + -f isvctl/configs/carbide/iam.yaml + +# Control plane validation +isvctl test run \ + -f isvctl/configs/templates/control-plane.yaml \ + -f isvctl/configs/carbide/control-plane.yaml + +# Network validation +CARBIDE_SITE_ID= isvctl test run \ + -f isvctl/configs/templates/network.yaml \ + -f isvctl/configs/carbide/network.yaml + +# Image registry validation +CARBIDE_SITE_ID= isvctl test run \ + -f isvctl/configs/templates/image-registry.yaml \ + -f isvctl/configs/carbide/image-registry.yaml + +# Bare metal instance validation +CARBIDE_SITE_ID= CARBIDE_OS_ID= CARBIDE_INSTANCE_TYPE= \ + isvctl test run \ + -f isvctl/configs/templates/bm.yaml \ + -f isvctl/configs/carbide/bm.yaml +``` + +## Authentication + +`carbidecli` handles authentication via its own config (`~/.carbide/config.yaml`) or environment variables: + +| Variable | Description | +|----------|-------------| +| `CARBIDE_TOKEN` | API bearer token | +| `CARBIDE_API_KEY` | NGC API key | +| `CARBIDE_ORG` | Organization/tenant name | + +## Required Environment Variables + +| Variable | Used by | Description | +|----------|---------|-------------| +| `CARBIDE_SITE_ID` | network, image-registry, bm | Site UUID for resource creation | +| `CARBIDE_OS_ID` | bm | Operating system UUID for instance provisioning | +| `CARBIDE_INSTANCE_TYPE` | bm | Instance type UUID (hardware profile) | +| `CARBIDE_VPC_ID` | bm (optional) | Pre-existing VPC to use | +| `CARBIDE_SSH_KEY_GROUP` | bm (optional) | SSH key group for instance access | + +## Extending for Platforms + +Carbide provides bare-metal infrastructure. Platforms (OpenShift, RHEL+Slurm, etc.) build on top: + +```bash +# OpenShift on Carbide +isvctl test run \ + -f isvctl/configs/templates/kaas.yaml \ + -f isvctl/configs/openshift/kaas-provision.yaml # AI + Carbide orchestration + -f isvctl/configs/openshift/kaas-overrides.yaml # OpenShift-specific checks + +# RHEL + Slurm on Carbide +isvctl test run \ + -f isvctl/configs/templates/bm.yaml \ + -f isvctl/configs/carbide/bm.yaml \ + -f isvctl/configs/slurm/bm-overrides.yaml # Slurm-specific checks +``` + +The platform layer handles: +- Orchestrating K8s/Slurm installation on Carbide bare metal +- Platform-specific operators (GPU Operator, Network Operator) +- Platform-specific validations (KubeVirt, DRA, ComputeDomains) diff --git a/isvctl/configs/carbide/bm.yaml b/isvctl/configs/carbide/bm.yaml new file mode 100644 index 00000000..6d1c2a56 --- /dev/null +++ b/isvctl/configs/carbide/bm.yaml @@ -0,0 +1,146 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# Carbide Bare Metal (BMaaS) - Layered Provider Configuration +# +# Supplies commands (carbidecli stubs) for the bm template. +# Tests bare-metal GPU instance lifecycle: launch, list, describe, +# reboot, NIM deploy, teardown, and verification. +# +# Mapping to template concepts: +# launch_instance → carbidecli instance create +# list_instances → carbidecli instance list +# describe_instance → carbidecli instance get +# reboot_instance → carbidecli instance reboot +# reinstall_instance → skip (not supported by Carbide) +# deploy_nim → shared NIM deploy script (SSH-based) +# teardown_nim → shared NIM teardown script (SSH-based) +# teardown → carbidecli instance delete +# verify_teardown → carbidecli instance get (verify terminated) +# +# Usage: +# isvctl test run \ +# -f isvctl/configs/templates/bm.yaml \ +# -f isvctl/configs/carbide/bm.yaml +# +# Environment: +# carbidecli handles auth via config or env vars: +# CARBIDE_TOKEN / CARBIDE_API_KEY — Authentication +# CARBIDE_ORG — Organization +# CARBIDE_SITE_ID — Site UUID (required) +# CARBIDE_OS_ID — Operating system UUID (required) +# CARBIDE_INSTANCE_TYPE — Instance type (required) + +version: "1.0" + +commands: + bm: + phases: ["setup", "test", "teardown"] + steps: + # Launch bare-metal GPU instance + - name: launch_instance + phase: setup + command: "python3 ../stubs/carbide/bm/launch_instance.py" + args: + - "--site-id" + - "{{ context.site_id }}" + - "--os-id" + - "{{ context.os_id }}" + - "--instance-type" + - "{{ context.instance_type }}" + timeout: 1200 + + # List instances - verify instance is visible + - name: list_instances + phase: test + command: "python3 ../stubs/carbide/bm/list_instances.py" + args: + - "--site-id" + - "{{ context.site_id }}" + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + timeout: 120 + + # Describe instance - test phase anchor for SSH/GPU validations + - name: describe_instance + phase: test + command: "python3 ../stubs/carbide/bm/describe_instance.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + timeout: 120 + + # Reboot instance and validate recovery + - name: reboot_instance + phase: test + command: "python3 ../stubs/carbide/bm/reboot_instance.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--public-ip" + - "{{steps.launch_instance.public_ip}}" + timeout: 1800 + + # Reinstall not supported by Carbide - always skipped + - name: reinstall_instance + phase: test + skip: true + command: "python3 ../stubs/carbide/bm/reinstall_instance.py" + timeout: 60 + + # Deploy NIM container via SSH (shared script) + # Requires NGC_API_KEY env var + - name: deploy_nim + phase: test + command: "python3 ../stubs/common/deploy_nim.py" + args: + - "--host" + - "{{steps.launch_instance.public_ip}}" + - "--key-file" + - "{{steps.launch_instance.key_file}}" + timeout: 1800 + + # Teardown NIM container via SSH (shared script) + - name: teardown_nim + phase: teardown + command: "python3 ../stubs/common/teardown_nim.py" + args: + - "--host" + - "{{steps.launch_instance.public_ip}}" + - "--key-file" + - "{{steps.launch_instance.key_file}}" + timeout: 120 + + # Teardown bare-metal instance + - name: teardown + phase: teardown + command: "python3 ../stubs/carbide/bm/teardown.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + timeout: 600 + + # Verify instance is terminated + - name: verify_teardown + phase: teardown + command: "python3 ../stubs/carbide/bm/verify_terminated.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + timeout: 120 + +# Provider-specific context +context: + site_id: "{{ env.CARBIDE_SITE_ID | default('') }}" + os_id: "{{ env.CARBIDE_OS_ID | default('') }}" + instance_type: "{{ env.CARBIDE_INSTANCE_TYPE | default('') }}" + +tests: + description: "Carbide bare metal (BMaaS): instance lifecycle, GPU, NIM deploy" diff --git a/isvctl/configs/carbide/control-plane.yaml b/isvctl/configs/carbide/control-plane.yaml new file mode 100644 index 00000000..c6b292f0 --- /dev/null +++ b/isvctl/configs/carbide/control-plane.yaml @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# Carbide Control Plane - Layered Provider Configuration +# +# Supplies commands (carbidecli stubs) for the control-plane template. +# Tests API health, SSH key lifecycle, and VPC lifecycle. +# +# Mapping to template concepts: +# API health → carbidecli tenant get, site list +# Access keys → SSH key group + SSH key CRUD +# Tenants → VPC CRUD (primary tenant-scoped resource) +# +# Usage: +# isvctl test run \ +# -f isvctl/configs/templates/control-plane.yaml \ +# -f isvctl/configs/carbide/control-plane.yaml +# +# Environment: +# carbidecli handles auth via config or env vars: +# CARBIDE_TOKEN / CARBIDE_API_KEY — Authentication +# CARBIDE_ORG — Organization +# CARBIDE_SITE_ID — Site UUID (required for VPC creation) + +version: "1.0" + +commands: + control_plane: + phases: ["setup", "test", "teardown"] + steps: + # ─── SETUP: API Health ───────────────────────────────────────── + - name: check_api + phase: setup + command: "python3 ../stubs/carbide/control-plane/check_api.py" + timeout: 120 + + # ─── SETUP: Create Resources ─────────────────────────────────── + - name: create_access_key + phase: setup + command: "python3 ../stubs/carbide/control-plane/create_access_key.py" + timeout: 120 + + - name: create_tenant + phase: setup + command: "python3 ../stubs/carbide/control-plane/create_tenant.py" + args: ["--site-id", "{{ context.site_id }}"] + timeout: 120 + + # ─── TEST: Access Key (SSH Key) Lifecycle ────────────────────── + - name: test_access_key + phase: test + command: "python3 ../stubs/carbide/control-plane/test_access_key.py" + args: + - "--access-key-id" + - "{{ steps.create_access_key.access_key_id }}" + timeout: 60 + + - name: disable_access_key + phase: test + command: "python3 ../stubs/carbide/control-plane/disable_access_key.py" + args: + - "--access-key-id" + - "{{ steps.create_access_key.access_key_id }}" + timeout: 60 + + - name: verify_key_rejected + phase: test + command: "python3 ../stubs/carbide/control-plane/verify_key_rejected.py" + args: + - "--access-key-id" + - "{{ steps.create_access_key.access_key_id }}" + timeout: 60 + + # ─── TEST: Tenant (VPC) Lifecycle ────────────────────────────── + - name: list_tenants + phase: test + command: "python3 ../stubs/carbide/control-plane/list_tenants.py" + args: + - "--target-group" + - "{{ steps.create_tenant.tenant_name }}" + timeout: 60 + + - name: get_tenant + phase: test + command: "python3 ../stubs/carbide/control-plane/get_tenant.py" + args: + - "--group-name" + - "{{ steps.create_tenant.tenant_name }}" + timeout: 60 + + # ─── TEARDOWN: Cleanup ───────────────────────────────────────── + - name: delete_access_key + phase: teardown + command: "python3 ../stubs/carbide/control-plane/delete_access_key.py" + args: + - "--username" + - "{{ steps.create_access_key.username }}" + timeout: 60 + output_schema: teardown + + - name: delete_tenant + phase: teardown + command: "python3 ../stubs/carbide/control-plane/delete_tenant.py" + args: + - "--group-name" + - "{{ steps.create_tenant.tenant_name }}" + timeout: 60 + output_schema: teardown + +# Provider-specific context +context: + site_id: "{{ env.CARBIDE_SITE_ID | default('') }}" + +tests: + description: "Carbide control plane: API health, SSH key lifecycle, VPC lifecycle" diff --git a/isvctl/configs/carbide/iam.yaml b/isvctl/configs/carbide/iam.yaml new file mode 100644 index 00000000..6061eed2 --- /dev/null +++ b/isvctl/configs/carbide/iam.yaml @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# Carbide IAM - Layered Provider Configuration +# +# Validates API token, scope permissions, and credential lifecycle. +# Carbide doesn't have user management (handled by NGC/Keycloak), +# so this validates token health + scope coverage instead. +# +# What it tests: +# - Token authentication (tenant get) +# - API read access (site list) +# - Scope coverage for all Carbide templates +# - Write access (create/delete temp SSH key) +# +# Usage: +# isvctl test run \ +# -f isvctl/configs/templates/iam.yaml \ +# -f isvctl/configs/carbide/iam.yaml +# +# Environment: +# TARGET_SCOPES — Granted API scopes (auto-populated by carbidecli) +# carbidecli handles auth via config or env vars + +version: "1.0" + +commands: + iam: + phases: ["setup", "test", "teardown"] + steps: + - name: create_user + phase: setup + command: "python3 ../stubs/carbide/iam/create_user.py" + timeout: 120 + + - name: test_credentials + phase: test + command: "python3 ../stubs/carbide/iam/test_credentials.py" + timeout: 120 + + - name: teardown + phase: teardown + command: "python3 ../stubs/carbide/iam/delete_user.py" + timeout: 60 + +tests: + description: "Carbide IAM: token validation, scope coverage, credential lifecycle" diff --git a/isvctl/configs/carbide/image-registry.yaml b/isvctl/configs/carbide/image-registry.yaml new file mode 100644 index 00000000..9b684e6a --- /dev/null +++ b/isvctl/configs/carbide/image-registry.yaml @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# Carbide Image Registry - Layered Provider Configuration +# +# Supplies commands (carbidecli stubs) for the image-registry template. +# Tests OS image upload, VM launch from image, install config CRUD, +# and BMaaS provisioning from images and configs. +# +# Mapping to template concepts: +# upload_image → carbidecli operating-system create (iPXE/kickstart type) +# launch_instance → carbidecli instance create using the OS +# crud_install_config → carbidecli operating-system get/list/update/delete lifecycle +# install_image_bm → carbidecli instance create with specific OS +# install_config_bm → same, proving OS+config works on BM +# teardown → delete instances and OS resources +# +# Usage: +# isvctl test run \ +# -f isvctl/configs/templates/image-registry.yaml \ +# -f isvctl/configs/carbide/image-registry.yaml +# +# Environment: +# carbidecli handles auth via config or env vars: +# CARBIDE_TOKEN / CARBIDE_API_KEY — Authentication +# CARBIDE_ORG — Organization +# CARBIDE_SITE_ID — Site UUID (required) + +version: "1.0" + +commands: + image_registry: + phases: ["setup", "test", "teardown"] + steps: + # ─── SETUP: Upload OS Image ────────────────────────────────────── + - name: upload_image + phase: setup + command: "python3 ../stubs/carbide/image-registry/upload_image.py" + args: + - "--site-id" + - "{{ context.site_id }}" + timeout: 600 + + # ─── TEST: Launch Instance from OS Image ───────────────────────── + - name: launch_instance + phase: test + command: "python3 ../stubs/carbide/image-registry/launch_instance.py" + args: + - "--os-id" + - "{{steps.upload_image.image_id}}" + - "--site-id" + - "{{ context.site_id }}" + timeout: 600 + + # ─── TEST: OS Install Config CRUD ───────────────────────────────── + - name: crud_install_config + phase: test + command: "python3 ../stubs/carbide/image-registry/crud_install_config.py" + args: + - "--os-id" + - "{{steps.upload_image.image_id}}" + - "--site-id" + - "{{ context.site_id }}" + timeout: 300 + + # ─── TEST: Install OS Image on BM ───────────────────────────────── + - name: install_image_bm + phase: test + command: "python3 ../stubs/carbide/image-registry/install_image_bm.py" + args: + - "--os-id" + - "{{steps.upload_image.image_id}}" + - "--site-id" + - "{{ context.site_id }}" + timeout: 600 + + # ─── TEST: Install Config on BM ─────────────────────────────────── + - name: install_config_bm + phase: test + command: "python3 ../stubs/carbide/image-registry/install_config_bm.py" + args: + - "--os-id" + - "{{steps.upload_image.image_id}}" + - "--site-id" + - "{{ context.site_id }}" + timeout: 600 + + # ─── TEARDOWN: Delete All Resources ─────────────────────────────── + - name: teardown + phase: teardown + command: "python3 ../stubs/carbide/image-registry/teardown.py" + args: + - "--site-id" + - "{{ context.site_id }}" + timeout: 300 + +# Provider-specific context +context: + site_id: "{{ env.CARBIDE_SITE_ID | default('') }}" + +tests: + description: "Carbide image registry: OS upload, VM launch, config CRUD, BM provisioning" diff --git a/isvctl/configs/carbide/network.yaml b/isvctl/configs/carbide/network.yaml new file mode 100644 index 00000000..113005e8 --- /dev/null +++ b/isvctl/configs/carbide/network.yaml @@ -0,0 +1,134 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# Carbide Network - Layered Provider Configuration +# +# Supplies commands (carbidecli stubs) for the network template. +# Tests VPC CRUD, subnet configuration, VPC isolation, NSG security, +# connectivity, and traffic routing. +# +# Mapping to template concepts: +# create_network → VPC + prefix + subnet creation +# vpc_crud → VPC create/get/update/delete lifecycle +# subnet_config → Multi-subnet creation and verification +# vpc_isolation → Two-VPC isolation verification +# security_blocking → NSG rule creation and verification +# connectivity_test → Subnet accessibility check +# traffic_validation → VPC + subnet routing verification +# teardown → Clean up all created resources +# +# Carbide network resources: +# VPC: carbidecli vpc create/get/list/delete +# Prefix: carbidecli vpc-prefix create/delete +# Subnet: carbidecli subnet create/get/list/delete +# NSG: carbidecli network-security-group create/get/list/delete +# +# Usage: +# isvctl test run \ +# -f isvctl/configs/templates/network.yaml \ +# -f isvctl/configs/carbide/network.yaml +# +# Environment: +# carbidecli handles auth via config or env vars: +# CARBIDE_TOKEN / CARBIDE_API_KEY — Authentication +# CARBIDE_ORG — Organization +# CARBIDE_SITE_ID — Site UUID (required) + +version: "1.0" + +commands: + network: + phases: ["setup", "test", "teardown"] + steps: + # ─── SETUP: Create shared VPC + subnet ─────────────────────────── + - name: create_network + phase: setup + command: "python3 ../stubs/carbide/network/create_vpc.py" + args: + - "--site-id" + - "{{ context.site_id }}" + timeout: 300 + + # ─── TEST: VPC CRUD lifecycle ──────────────────────────────────── + - name: vpc_crud + phase: test + command: "python3 ../stubs/carbide/network/vpc_crud_test.py" + args: + - "--site-id" + - "{{ context.site_id }}" + timeout: 120 + output_schema: vpc_crud + + # ─── TEST: Subnet configuration ────────────────────────────────── + - name: subnet_config + phase: test + command: "python3 ../stubs/carbide/network/subnet_test.py" + args: + - "--site-id" + - "{{ context.site_id }}" + timeout: 120 + output_schema: subnet_config + + # ─── TEST: VPC isolation ────────────────────────────────────────── + - name: vpc_isolation + phase: test + command: "python3 ../stubs/carbide/network/isolation_test.py" + args: + - "--site-id" + - "{{ context.site_id }}" + timeout: 120 + output_schema: vpc_isolation + + # ─── TEST: NSG security blocking ───────────────────────────────── + - name: security_blocking + phase: test + command: "python3 ../stubs/carbide/network/security_test.py" + args: + - "--site-id" + - "{{ context.site_id }}" + timeout: 120 + output_schema: security_blocking + + # ─── TEST: Connectivity ─────────────────────────────────────────── + - name: connectivity_test + phase: test + command: "python3 ../stubs/carbide/network/test_connectivity.py" + args: + - "--site-id" + - "{{ context.site_id }}" + timeout: 120 + output_schema: connectivity_result + + # ─── TEST: Traffic validation ───────────────────────────────────── + - name: traffic_validation + phase: test + command: "python3 ../stubs/carbide/network/traffic_test.py" + args: + - "--site-id" + - "{{ context.site_id }}" + timeout: 120 + output_schema: traffic_flow + + # ─── TEARDOWN: Clean up all resources ───────────────────────────── + - name: teardown + phase: teardown + command: "python3 ../stubs/carbide/network/teardown.py" + args: + - "--site-id" + - "{{ context.site_id }}" + timeout: 300 + output_schema: teardown + +# Provider-specific context +context: + site_id: "{{ env.CARBIDE_SITE_ID | default('') }}" + +tests: + description: "Carbide network: VPC CRUD, subnet config, isolation, NSG security, connectivity" diff --git a/isvctl/configs/openshift/arm.yaml b/isvctl/configs/openshift/arm.yaml new file mode 100644 index 00000000..97c5f81f --- /dev/null +++ b/isvctl/configs/openshift/arm.yaml @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# OpenShift ARM/aarch64 Platform Validation +# +# Read-only checks for ARM-specific platform features. +# All tests skip gracefully on x86_64 clusters. +# +# Validates: +# - aarch64 architecture on nodes +# - SMMUv3 (System MMU for device isolation) +# - GICv4.1 (Generic Interrupt Controller for VFIO passthrough) +# - Hugepages configuration +# - NUMA topology +# +# Usage: +# isvctl test run -f isvctl/configs/openshift/arm.yaml + +version: "1.0" + +commands: + openshift: + phases: ["setup", "test", "teardown"] + steps: + - name: arch_check + phase: test + command: "python3 ../stubs/openshift/arm/arch_check.py" + timeout: 60 + + - name: smmu_check + phase: test + command: "python3 ../stubs/openshift/arm/smmu_check.py" + timeout: 120 + + - name: gicv4_check + phase: test + command: "python3 ../stubs/openshift/arm/gicv4_check.py" + timeout: 120 + + - name: hugepage_check + phase: test + command: "python3 ../stubs/openshift/arm/hugepage_check.py" + timeout: 60 + + - name: topology_check + phase: test + command: "python3 ../stubs/openshift/arm/topology_check.py" + timeout: 120 + +tests: + platform: openshift + description: "OpenShift ARM/aarch64 platform validation" + + validations: + architecture: + checks: + - StepSuccessCheck: + step: arch_check + smmu: + checks: + - StepSuccessCheck: + step: smmu_check + gic: + checks: + - StepSuccessCheck: + step: gicv4_check + hugepages: + checks: + - StepSuccessCheck: + step: hugepage_check + topology: + checks: + - StepSuccessCheck: + step: topology_check + + exclude: + markers: [] diff --git a/isvctl/configs/openshift/computedomain.yaml b/isvctl/configs/openshift/computedomain.yaml new file mode 100644 index 00000000..a9afb3e1 --- /dev/null +++ b/isvctl/configs/openshift/computedomain.yaml @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# OpenShift ComputeDomain (IMEX) Validation +# +# Validates ComputeDomain CRD and IMEX channel functionality: +# - ComputeDomain CRD exists (installed by GPU Operator in DRA mode) +# - Create a ComputeDomain spanning multiple GPU nodes +# - Verify IMEX channels established +# - Tray allocation test +# - Multi-job scheduling test +# - NVLink NCCL bandwidth across ComputeDomain +# +# Prerequisites: +# - GPU Operator in DRA mode (run dra.yaml first) +# - NVL72 rack with NVLink fabric +# +# Usage: +# isvctl test run -f isvctl/configs/openshift/computedomain.yaml + +version: "1.0" + +commands: + openshift: + phases: ["setup", "test", "teardown"] + steps: + - name: check_crd + phase: setup + command: "python3 ../stubs/openshift/computedomain/check_crd.py" + timeout: 60 + + - name: create_domain + phase: setup + command: "python3 ../stubs/openshift/computedomain/create_domain.py" + timeout: 300 + + - name: tray_allocation_test + phase: test + command: "python3 ../stubs/openshift/computedomain/tray_allocation_test.py" + timeout: 120 + + - name: multi_job_test + phase: test + command: "python3 ../stubs/openshift/computedomain/multi_job_test.py" + timeout: 300 + + - name: nccl_nvlink_test + phase: test + command: "python3 ../stubs/openshift/computedomain/nccl_nvlink_test.py" + timeout: 600 + + - name: teardown + phase: teardown + command: "python3 ../stubs/openshift/computedomain/teardown.py" + timeout: 120 + +tests: + platform: openshift + description: "OpenShift ComputeDomain (IMEX) validation" + + validations: + crd: + checks: + - StepSuccessCheck: + step: check_crd + domain: + checks: + - StepSuccessCheck: + step: create_domain + - FieldExistsCheck: + step: create_domain + fields: ["domain_name", "imex_channels_ready"] + tray: + checks: + - StepSuccessCheck: + step: tray_allocation_test + multi_job: + checks: + - StepSuccessCheck: + step: multi_job_test + nccl: + checks: + - StepSuccessCheck: + step: nccl_nvlink_test + + exclude: + markers: [] diff --git a/isvctl/configs/openshift/dra.yaml b/isvctl/configs/openshift/dra.yaml new file mode 100644 index 00000000..f554d935 --- /dev/null +++ b/isvctl/configs/openshift/dra.yaml @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# OpenShift DRA (Dynamic Resource Allocation) Validation +# +# Validates GPU Operator in DRA mode: +# - DRA driver pods running +# - ResourceClass exists for GPUs +# - ResourceClaim allocation works +# - SCC compatibility with DRA +# +# Prerequisites: +# - GPU Operator switched to DRA mode (ClusterPolicy devicePlugin.enabled=false) +# - OpenShift 4.17+ with DRA feature gate +# +# Usage: +# isvctl test run -f isvctl/configs/openshift/dra.yaml + +version: "1.0" + +commands: + openshift: + phases: ["setup", "test", "teardown"] + steps: + - name: dra_setup + phase: setup + command: "python3 ../stubs/openshift/dra/dra_setup.py" + timeout: 300 + + - name: dra_claim_test + phase: test + command: "python3 ../stubs/openshift/dra/dra_claim_test.py" + timeout: 300 + + - name: dra_scc_check + phase: test + command: "python3 ../stubs/openshift/dra/dra_scc_check.py" + timeout: 120 + + - name: teardown + phase: teardown + command: "python3 ../stubs/openshift/dra/teardown.py" + timeout: 60 + +tests: + platform: openshift + description: "OpenShift DRA mode validation (GPU Operator)" + + validations: + dra_health: + checks: + - StepSuccessCheck: + step: dra_setup + - FieldExistsCheck: + step: dra_setup + fields: ["driver_ready", "resource_class_exists"] + + dra_claim: + checks: + - StepSuccessCheck: + step: dra_claim_test + + dra_scc: + checks: + - StepSuccessCheck: + step: dra_scc_check + + exclude: + markers: [] diff --git a/isvctl/configs/openshift/gpu-health.yaml b/isvctl/configs/openshift/gpu-health.yaml new file mode 100644 index 00000000..cee67d10 --- /dev/null +++ b/isvctl/configs/openshift/gpu-health.yaml @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# OpenShift GPU Health & Observability Validation +# +# Read-only checks against existing GPU infrastructure: +# - DCGM Exporter running and exposing metrics +# - DCGM metrics content validation +# - Node Health Check (NHC) operator +# - Prometheus GPU metrics freshness +# +# Usage: +# isvctl test run -f isvctl/configs/openshift/gpu-health.yaml + +version: "1.0" + +commands: + openshift: + phases: ["setup", "test", "teardown"] + steps: + - name: dcgm_exporter_check + phase: test + command: "python3 ../stubs/openshift/gpu-health/dcgm_exporter_check.py" + timeout: 120 + + - name: dcgm_metrics_validate + phase: test + command: "python3 ../stubs/openshift/gpu-health/dcgm_metrics_validate.py" + timeout: 120 + + - name: nhc_operator_check + phase: test + command: "python3 ../stubs/openshift/gpu-health/nhc_operator_check.py" + timeout: 120 + + - name: prometheus_gpu_metrics + phase: test + command: "python3 ../stubs/openshift/gpu-health/prometheus_gpu_metrics.py" + timeout: 120 + +tests: + platform: openshift + description: "OpenShift GPU health and observability validation" + + validations: + dcgm_exporter: + checks: + - StepSuccessCheck: + step: dcgm_exporter_check + - FieldExistsCheck: + step: dcgm_exporter_check + fields: ["exporter_running", "metrics_endpoint"] + + dcgm_metrics: + checks: + - StepSuccessCheck: + step: dcgm_metrics_validate + + nhc: + checks: + - StepSuccessCheck: + step: nhc_operator_check + + prometheus: + checks: + - StepSuccessCheck: + step: prometheus_gpu_metrics + + exclude: + markers: [] diff --git a/isvctl/configs/openshift/hosted-provision.yaml b/isvctl/configs/openshift/hosted-provision.yaml new file mode 100644 index 00000000..96ebb0d0 --- /dev/null +++ b/isvctl/configs/openshift/hosted-provision.yaml @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# OpenShift Hosted Control Plane Provisioning +# +# Deploys a hosted OpenShift cluster from the management cluster using +# Cluster API (CAPI) Provider for Carbide. The flow is: +# +# 1. Deploy CAPI Provider for Carbide on management cluster +# 2. Provision BareMetalHosts via Carbide (6 GPU nodes) +# 3. Create Keycloak realm for hosted cluster auth +# 4. Deploy hosted OpenShift cluster with BareMetalHosts as workers +# 5. Configure GPU Operator, Network Operator, Virt on hosted cluster +# 6. Export hosted cluster kubeconfig +# +# After provisioning, run the same validation tests targeting the +# hosted cluster by setting KUBECONFIG to the hosted cluster's kubeconfig. +# +# Prerequisites: +# - Management cluster running (from kaas-provision.yaml) +# - CAPI Provider for Carbide installed +# - Carbide API access for BareMetalHost provisioning +# - Keycloak running on management cluster (from iam.yaml) +# +# Usage: +# # Provision hosted cluster +# isvctl test run -f isvctl/configs/openshift/hosted-provision.yaml +# +# # Then validate hosted cluster +# KUBECONFIG=/tmp/hosted-kubeconfig \ +# isvctl test run \ +# -f isvctl/configs/templates/kaas.yaml \ +# -f isvctl/configs/openshift/kaas.yaml +# +# Environment: +# CARBIDE_SITE_ID — Site UUID for BareMetalHost provisioning +# CARBIDE_INSTANCE_TYPE — Instance type UUID for hosted worker nodes +# HOSTED_CLUSTER_NAME — Hosted cluster name (default: ncp-hosted) +# HOSTED_WORKER_COUNT — Number of worker nodes (default: 6) +# HOSTED_BASE_DOMAIN — Base DNS domain (default: from management cluster) +# KEYCLOAK_NAMESPACE — Keycloak namespace on mgmt cluster (default: ncp-keycloak) +# KEYCLOAK_REALM — Realm for hosted cluster (default: ncp-hosted) +# TEARDOWN_ENABLED — Set to "true" to enable deprovision + +version: "1.0" + +commands: + openshift: + phases: ["setup", "test", "teardown"] + steps: + - name: deploy_capi_provider + phase: setup + command: "python3 ../stubs/openshift/hosted/deploy_capi_provider.py" + timeout: 600 + + - name: provision_baremetal_hosts + phase: setup + command: "python3 ../stubs/openshift/hosted/provision_baremetal_hosts.py" + timeout: 3600 + + - name: create_hosted_realm + phase: setup + command: "python3 ../stubs/openshift/hosted/create_hosted_realm.py" + timeout: 120 + + - name: deploy_hosted_cluster + phase: setup + command: "python3 ../stubs/openshift/hosted/deploy_hosted_cluster.py" + timeout: 7200 + + - name: configure_hosted_cluster + phase: setup + command: "python3 ../stubs/openshift/hosted/configure_hosted_cluster.py" + timeout: 900 + + - name: verify_hosted_cluster + phase: test + command: "python3 ../stubs/openshift/hosted/verify_hosted_cluster.py" + timeout: 300 + + - name: teardown + phase: teardown + command: "python3 ../stubs/openshift/hosted/teardown.py" + timeout: 1800 + +tests: + platform: openshift + description: "OpenShift Hosted Control Plane provisioning via CAPI + Carbide" + + validations: + capi: + checks: + - StepSuccessCheck: + step: deploy_capi_provider + baremetal: + checks: + - StepSuccessCheck: + step: provision_baremetal_hosts + - FieldExistsCheck: + step: provision_baremetal_hosts + fields: ["hosts_ready", "host_count"] + hosted_cluster: + checks: + - StepSuccessCheck: + step: deploy_hosted_cluster + - FieldExistsCheck: + step: deploy_hosted_cluster + fields: ["cluster_ready", "kubeconfig_path"] + hosted_verification: + checks: + - StepSuccessCheck: + step: verify_hosted_cluster + + exclude: + markers: [] diff --git a/isvctl/configs/openshift/iam.yaml b/isvctl/configs/openshift/iam.yaml new file mode 100644 index 00000000..c215832d --- /dev/null +++ b/isvctl/configs/openshift/iam.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# OpenShift IAM Validation (Keycloak via Red Hat Build of Keycloak Operator) +# +# Tests user lifecycle: deploy Keycloak (idempotent), create test user, +# verify oc login with user credentials, check RBAC, clean up user. +# +# Prerequisites: +# - OpenShift cluster with cluster-admin access +# - Access to OperatorHub (redhat-operators catalog) +# - oc and kubectl CLI configured +# +# The create_user step is idempotent: it deploys the RHBK operator and +# Keycloak instance if not already present, configures the OAuth IdP, +# then creates a test user. Subsequent runs skip the deployment. +# +# Usage: +# isvctl test run \ +# -f isvctl/configs/templates/iam.yaml \ +# -f isvctl/configs/openshift/iam.yaml +# +# Environment: +# KEYCLOAK_NAMESPACE — Namespace for Keycloak (default: ncp-keycloak) +# KEYCLOAK_REALM — Realm name (default: ncp-validation) +# TEST_USERNAME — Test user name (default: ncp-test-user) +# TEST_PASSWORD — Test user password (default: ncp-test-pass) + +version: "1.0" + +commands: + iam: + phases: ["setup", "test", "teardown"] + steps: + - name: create_user + phase: setup + command: "python3 ../stubs/openshift/iam/create_user.py" + timeout: 600 # includes Keycloak deployment on first run + + - name: test_credentials + phase: test + command: "python3 ../stubs/openshift/iam/test_credentials.py" + timeout: 120 + + - name: teardown + phase: teardown + command: "python3 ../stubs/openshift/iam/teardown.py" + timeout: 120 diff --git a/isvctl/configs/openshift/kaas-overrides.yaml b/isvctl/configs/openshift/kaas-overrides.yaml new file mode 100644 index 00000000..8cfc5d09 --- /dev/null +++ b/isvctl/configs/openshift/kaas-overrides.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# OpenShift KaaS Overrides +# +# Merge on top of the template + provider config to customize for your cluster. +# +# Usage: +# isvctl test run \ +# -f isvctl/configs/templates/kaas.yaml \ +# -f isvctl/configs/openshift/kaas.yaml \ +# -f isvctl/configs/openshift/kaas-overrides.yaml +# +# Or override individual values: +# isvctl test run ... --set context.node_count=21 --set context.total_gpus=72 + +# Uncomment and adjust for your cluster: + +# context: +# node_count: "21" # 3 control + 18 GPU nodes +# gpu_per_node: "4" # NVL72: 4 GPUs per tray +# total_gpus: "72" # 18 × 4 +# gpu_node_count: "18" +# min_nccl_bw: "400" # NVLink: 400+ GB/s +# driver_version: "550.127.05" + +# Exclude slow workloads for quick validation: +# tests: +# exclude: +# markers: +# - slow +# - workload diff --git a/isvctl/configs/openshift/kaas-provision.yaml b/isvctl/configs/openshift/kaas-provision.yaml new file mode 100644 index 00000000..a366f90e --- /dev/null +++ b/isvctl/configs/openshift/kaas-provision.yaml @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# OpenShift KaaS Provisioning on Carbide +# +# Orchestrates Assisted Installer + Carbide to provision an OpenShift cluster +# on bare-metal infrastructure. The flow is: +# +# 1. Create cluster in Assisted Installer (aicli) +# 2. Collect iPXE config with discovery token +# 3. Create OperatingSystem in Carbide with iPXE config +# 4. Batch-create bare-metal instances with OS + InstanceType +# 5. Collect instance UUIDs + MAC addresses +# 6. Monitor hosts in AI, match to instances, approve +# 7. Monitor OpenShift installation, collect kubeconfig +# 8. Day-2: GPU Operator, Network Operator, etc. +# +# Supports pre-existing Carbide resources (Tenant, VPC, Instance Type) +# via environment variables. +# +# Usage: +# # Layered: template validations + OpenShift provisioning +# isvctl test run \ +# -f isvctl/configs/templates/kaas.yaml \ +# -f isvctl/configs/openshift/kaas-provision.yaml +# +# # With OpenShift-specific overrides +# isvctl test run \ +# -f isvctl/configs/templates/kaas.yaml \ +# -f isvctl/configs/openshift/kaas-provision.yaml \ +# -f isvctl/configs/openshift/kaas-overrides.yaml +# +# Carbide Environment (required): +# CARBIDE_TOKEN / CARBIDE_API_KEY — Authentication +# CARBIDE_ORG — Organization +# CARBIDE_SITE_ID — Site UUID +# CARBIDE_INSTANCE_TYPE — Instance type UUID +# +# Carbide Environment (optional, use pre-existing): +# CARBIDE_VPC_ID — Pre-existing VPC UUID +# CARBIDE_SUBNET_ID — Pre-existing subnet UUID +# CARBIDE_SSH_KEY_GROUP — SSH key group UUID +# +# Assisted Installer Environment (required): +# AI_OFFLINETOKEN — Assisted Installer offline token (SaaS) +# AI_URL — Assisted Installer URL (on-prem, overrides SaaS) +# OCP_PULL_SECRET — OpenShift pull secret (JSON or file path) +# +# OpenShift Environment (optional): +# OCP_VERSION — OpenShift version (default: 4.18) +# OCP_BASE_DOMAIN — Base DNS domain (default: example.com) +# CARBIDE_INSTANCE_COUNT — Number of instances (default: 3) +# TEARDOWN_ENABLED — Set to "true" to enable deprovision + +version: "1.0" + +commands: + kubernetes: + phases: ["setup", "test", "teardown"] + steps: + - name: provision_cluster + phase: setup + command: "python3 ../stubs/openshift/kaas/provision.py" + timeout: 7200 # 2 hours for full bare-metal provisioning + OpenShift install + + - name: teardown_cluster + phase: teardown + command: "python3 ../stubs/openshift/kaas/deprovision.py" + timeout: 1800 # 30 minutes for cleanup + +# Context overrides for OpenShift on Carbide defaults +context: + node_count: "3" + gpu_per_node: "4" + total_gpus: "12" + gpu_node_count: "3" + runtime_class: "" + gpu_operator_ns: "nvidia-gpu-operator" + min_nccl_bw: "100" + +tests: + description: "OpenShift on Carbide: bare-metal provisioned cluster validation" diff --git a/isvctl/configs/openshift/kaas.yaml b/isvctl/configs/openshift/kaas.yaml new file mode 100644 index 00000000..76e449ff --- /dev/null +++ b/isvctl/configs/openshift/kaas.yaml @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# OpenShift KaaS Validation - Pre-existing Cluster +# +# Validates an existing OpenShift cluster with GPU Operator. +# Does not provision or deprovision — assumes the cluster is already running. +# +# Usage: +# # Layered: template validations + OpenShift inventory +# isvctl test run \ +# -f isvctl/configs/templates/kaas.yaml \ +# -f isvctl/configs/openshift/kaas.yaml +# +# # With overrides for specific cluster +# isvctl test run \ +# -f isvctl/configs/templates/kaas.yaml \ +# -f isvctl/configs/openshift/kaas.yaml \ +# --set context.node_count=21 --set context.total_gpus=72 +# +# Requirements: +# - oc CLI configured and authenticated +# - GPU Operator installed on the cluster +# +# Environment: +# KUBECONFIG — Path to kubeconfig (default: ~/.kube/config) + +version: "1.0" + +commands: + kubernetes: + phases: ["setup", "test", "teardown"] + steps: + - name: provision_cluster + phase: setup + command: "bash ../stubs/openshift/kaas/setup.sh" + timeout: 120 + + - name: teardown_cluster + phase: teardown + command: "bash ../stubs/openshift/kaas/teardown.sh" + timeout: 60 + +# OpenShift-specific defaults +context: + runtime_class: "" + gpu_operator_ns: "nvidia-gpu-operator" + +tests: + description: "OpenShift GPU cluster validation (pre-existing cluster)" diff --git a/isvctl/configs/openshift/machineset.yaml b/isvctl/configs/openshift/machineset.yaml new file mode 100644 index 00000000..d7862ed7 --- /dev/null +++ b/isvctl/configs/openshift/machineset.yaml @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# OpenShift MachineSet Scaling Validation (Carbide Machine API Provider) +# +# Validates dynamic GPU node provisioning via MachineSet: +# 1. Verify Machine API configured with Carbide provider +# 2. Create MachineSet targeting GPU instance type +# 3. Wait for machines to provision and join as nodes +# 4. Run single-node GPU workload (proves GPU scheduling) +# 5. Scale up: trigger multi-node workload requiring >2 nodes +# 6. Scale down: reduce replicas, verify deprovisioning +# +# Prerequisites: +# - OpenShift cluster with Machine API +# - Cloud Provider for Carbide installed +# - Machine API Provider for Carbide installed +# - GPU Operator installed +# - kubectl and oc CLI configured +# +# Usage: +# isvctl test run -f isvctl/configs/openshift/machineset.yaml +# +# Environment: +# CARBIDE_INSTANCE_TYPE — Instance type for GPU machines +# CARBIDE_SITE_ID — Site UUID +# MACHINESET_MIN_REPLICAS — Initial replicas (default: 2) +# MACHINESET_MAX_REPLICAS — Max replicas for autoscaler (default: 6) + +version: "1.0" + +commands: + openshift: + phases: ["setup", "test", "teardown"] + steps: + - name: verify_machine_api + phase: setup + command: "python3 ../stubs/openshift/machineset/verify_machine_api.py" + timeout: 120 + + - name: create_machineset + phase: setup + command: "python3 ../stubs/openshift/machineset/create_machineset.py" + timeout: 1800 + + - name: test_gpu_workload + phase: test + command: "python3 ../stubs/openshift/machineset/test_gpu_workload.py" + timeout: 600 + + - name: test_scale_up + phase: test + command: "python3 ../stubs/openshift/machineset/test_scale_up.py" + timeout: 1800 + + - name: test_scale_down + phase: test + command: "python3 ../stubs/openshift/machineset/test_scale_down.py" + timeout: 900 + + - name: teardown + phase: teardown + command: "python3 ../stubs/openshift/machineset/teardown.py" + timeout: 900 + +tests: + platform: openshift + description: "OpenShift MachineSet scaling validation (Carbide Machine API)" + + validations: + machine_api: + checks: + - StepSuccessCheck: + step: verify_machine_api + - FieldExistsCheck: + step: verify_machine_api + fields: ["provider_configured", "machine_api_ready"] + + provisioning: + checks: + - StepSuccessCheck: + step: create_machineset + - FieldExistsCheck: + step: create_machineset + fields: ["replicas_ready", "nodes_joined"] + + gpu_workload: + checks: + - StepSuccessCheck: + step: test_gpu_workload + + scale_up: + checks: + - StepSuccessCheck: + step: test_scale_up + - FieldExistsCheck: + step: test_scale_up + fields: ["scaled_to", "nodes_after"] + + scale_down: + checks: + - StepSuccessCheck: + step: test_scale_down + - FieldExistsCheck: + step: test_scale_down + fields: ["scaled_to", "nodes_after"] + + exclude: + markers: [] diff --git a/isvctl/configs/openshift/network.yaml b/isvctl/configs/openshift/network.yaml new file mode 100644 index 00000000..ae69c3d4 --- /dev/null +++ b/isvctl/configs/openshift/network.yaml @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# OpenShift GPU Network Validation +# +# Validates the in-cluster GPU networking stack: +# - NVIDIA Network Operator (MOFED drivers, RDMA device plugin) +# - SR-IOV Operator (VF provisioning for InfiniBand/RoCE) +# - NetworkPolicy enforcement (OVN-Kubernetes) +# - Multus secondary interfaces +# - EgressFirewall rules +# - Multi-node NCCL AllReduce over RDMA +# +# This is NOT the infrastructure-level network template (VPC CRUD). +# It tests the Kubernetes network fabric for GPU workloads. +# +# Requirements: +# - OpenShift cluster with GPU nodes +# - NVIDIA Network Operator installed +# - SR-IOV Network Operator installed (for InfiniBand/RoCE) +# - kubectl and oc CLI configured +# +# Usage: +# isvctl test run -f isvctl/configs/openshift/network.yaml +# +# Environment: +# K8S_NAMESPACE — Test namespace (default: ncp-network-validation) + +version: "1.0" + +commands: + openshift: + phases: ["setup", "test", "teardown"] + steps: + - name: setup_namespaces + phase: setup + command: "python3 ../stubs/openshift/network/setup_namespaces.py" + timeout: 60 + + - name: network_operator_check + phase: test + command: "python3 ../stubs/openshift/network/network_operator_check.py" + timeout: 120 + + - name: sriov_check + phase: test + command: "python3 ../stubs/openshift/network/sriov_check.py" + timeout: 120 + + - name: network_policy_test + phase: test + command: "python3 ../stubs/openshift/network/network_policy_test.py" + timeout: 120 + + - name: multus_test + phase: test + command: "python3 ../stubs/openshift/network/multus_test.py" + timeout: 120 + + - name: rdma_test + phase: test + command: "python3 ../stubs/openshift/network/rdma_test.py" + timeout: 120 + + - name: nccl_multinode_test + phase: test + command: "python3 ../stubs/openshift/network/nccl_multinode_test.py" + timeout: 600 + + - name: teardown + phase: teardown + command: "python3 ../stubs/openshift/network/teardown.py" + timeout: 60 + +tests: + platform: openshift + description: "OpenShift GPU network validation (Network Operator, SR-IOV, RDMA, NCCL)" + + validations: + network_operator: + checks: + - StepSuccessCheck: + step: network_operator_check + - FieldExistsCheck: + step: network_operator_check + fields: ["mofed_ready", "rdma_device_plugin_ready"] + + sriov: + checks: + - StepSuccessCheck: + step: sriov_check + - FieldExistsCheck: + step: sriov_check + fields: ["sriov_operator_ready", "vf_count"] + + network_policy: + checks: + - StepSuccessCheck: + step: network_policy_test + + multus: + checks: + - StepSuccessCheck: + step: multus_test + + rdma: + checks: + - StepSuccessCheck: + step: rdma_test + + nccl: + checks: + - StepSuccessCheck: + step: nccl_multinode_test + + exclude: + markers: [] diff --git a/isvctl/configs/openshift/security.yaml b/isvctl/configs/openshift/security.yaml new file mode 100644 index 00000000..c00ce676 --- /dev/null +++ b/isvctl/configs/openshift/security.yaml @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# OpenShift Security & Compliance Validation +# +# Read-only checks against existing nodes: +# - SELinux enforcing mode +# - Secure Boot enabled +# - FIPS mode enabled +# - Compliance Operator scan results +# +# Usage: +# isvctl test run -f isvctl/configs/openshift/security.yaml + +version: "1.0" + +commands: + openshift: + phases: ["setup", "test", "teardown"] + steps: + - name: selinux_check + phase: test + command: "python3 ../stubs/openshift/security/selinux_check.py" + timeout: 120 + + - name: secure_boot_check + phase: test + command: "python3 ../stubs/openshift/security/secure_boot_check.py" + timeout: 120 + + - name: fips_check + phase: test + command: "python3 ../stubs/openshift/security/fips_check.py" + timeout: 120 + + - name: compliance_scan + phase: test + command: "python3 ../stubs/openshift/security/compliance_scan.py" + timeout: 300 + +tests: + platform: openshift + description: "OpenShift security and compliance validation" + + validations: + selinux: + checks: + - StepSuccessCheck: + step: selinux_check + secure_boot: + checks: + - StepSuccessCheck: + step: secure_boot_check + fips: + checks: + - StepSuccessCheck: + step: fips_check + compliance: + checks: + - StepSuccessCheck: + step: compliance_scan + + exclude: + markers: [] diff --git a/isvctl/configs/openshift/storage.yaml b/isvctl/configs/openshift/storage.yaml new file mode 100644 index 00000000..e0e68648 --- /dev/null +++ b/isvctl/configs/openshift/storage.yaml @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# OpenShift Storage Validation (ODF on NVMe) +# +# Validates OpenShift Data Foundation deployment and functionality: +# - ODF operator installed (idempotent) +# - Local NVMe disk discovery +# - StorageCluster creation +# - StorageClass provisioning (CephRBD, CephFS) +# - PVC binding and pod mount +# +# This is OpenShift-specific — no upstream template equivalent. +# +# Requirements: +# - OpenShift cluster with cluster-admin access +# - Nodes with local NVMe disks (8 x 3.5TB per node typical) +# - kubectl and oc CLI configured +# +# Usage: +# isvctl test run -f isvctl/configs/openshift/storage.yaml +# +# Environment: +# ODF_NAMESPACE — ODF namespace (default: openshift-storage) +# ODF_DISK_COUNT — Min NVMe disks per node (default: 1) +# ODF_STORAGE_NODES — Comma-separated node names for ODF (default: auto-detect) + +version: "1.0" + +commands: + openshift: + phases: ["setup", "test", "teardown"] + steps: + - name: deploy_odf + phase: setup + command: "python3 ../stubs/openshift/storage/deploy_odf.py" + timeout: 900 + + - name: verify_storage_classes + phase: test + command: "python3 ../stubs/openshift/storage/verify_storage_classes.py" + timeout: 120 + + - name: test_pvc_binding + phase: test + command: "python3 ../stubs/openshift/storage/test_pvc_binding.py" + timeout: 300 + + - name: test_pod_mount + phase: test + command: "python3 ../stubs/openshift/storage/test_pod_mount.py" + timeout: 300 + + - name: teardown + phase: teardown + command: "python3 ../stubs/openshift/storage/teardown.py" + timeout: 120 + +tests: + platform: openshift + description: "OpenShift storage validation (ODF on NVMe)" + + validations: + odf_deployment: + checks: + - StepSuccessCheck: + step: deploy_odf + - FieldExistsCheck: + step: deploy_odf + fields: ["odf_ready", "disks_discovered"] + + storage_classes: + checks: + - StepSuccessCheck: + step: verify_storage_classes + - FieldExistsCheck: + step: verify_storage_classes + fields: ["rbd_class_exists", "cephfs_class_exists"] + + pvc: + checks: + - StepSuccessCheck: + step: test_pvc_binding + + pod_mount: + checks: + - StepSuccessCheck: + step: test_pod_mount + + exclude: + markers: [] diff --git a/isvctl/configs/openshift/vm.yaml b/isvctl/configs/openshift/vm.yaml new file mode 100644 index 00000000..7b9267a1 --- /dev/null +++ b/isvctl/configs/openshift/vm.yaml @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# OpenShift VM Validation (KubeVirt + GPU Passthrough) +# +# Implements the VM template using OpenShift Virtualization (KubeVirt) +# with GPU passthrough via VFIO. Creates a VirtualMachine with a GPU +# device, verifies GPU access, reboots, and optionally deploys NIM. +# +# Prerequisites: +# - OpenShift Virtualization operator installed +# - GPU nodes with VFIO configured (via GPU Operator hostDevicePlugin) +# - kubectl, oc, and virtctl CLI configured +# +# Usage: +# isvctl test run \ +# -f isvctl/configs/templates/vm.yaml \ +# -f isvctl/configs/openshift/vm.yaml +# +# Environment: +# VM_NAMESPACE — VM namespace (default: ncp-vm-validation) +# VM_GPU_COUNT — GPUs to passthrough (default: 1) +# VM_DISK_SIZE — Root disk size (default: 50Gi) +# VM_MEMORY — VM memory (default: 16Gi) +# VM_CPUS — VM CPU cores (default: 8) +# VM_IMAGE_URL — Cloud image URL (default: RHEL 9 qcow2) +# VM_SSH_PUBKEY — SSH public key for VM access + +version: "1.0" + +commands: + vm: + phases: ["setup", "test", "teardown"] + steps: + - name: launch_instance + phase: setup + command: "python3 ../stubs/openshift/vm/launch_instance.py" + timeout: 600 + + - name: list_instances + phase: test + command: "python3 ../stubs/openshift/vm/list_instances.py" + timeout: 60 + + - name: reboot_instance + phase: test + command: "python3 ../stubs/openshift/vm/reboot_instance.py" + timeout: 300 + + - name: deploy_nim + phase: test + command: "python3 ../stubs/openshift/vm/deploy_nim.py" + timeout: 1800 + skip: true # Enable with --set commands.vm.steps[3].skip=false + + - name: teardown_nim + phase: teardown + command: "echo '{\"success\": true, \"platform\": \"vm\"}'" + timeout: 30 + + - name: teardown + phase: teardown + command: "python3 ../stubs/openshift/vm/teardown.py" + timeout: 300 + +context: + expected_gpus: "1" + expected_os: "rhel" + max_reboot_uptime: "600" diff --git a/isvctl/configs/stubs/carbide/bm/describe_instance.py b/isvctl/configs/stubs/carbide/bm/describe_instance.py new file mode 100644 index 00000000..f04202ca --- /dev/null +++ b/isvctl/configs/stubs/carbide/bm/describe_instance.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Describe a running Carbide bare-metal instance. + +Lightweight test-phase step that fetches current instance state and +passes through SSH connection info. Validations (SSH, GPU, host OS) +bind to this step so they run in the test phase rather than setup. + +Usage: + python describe_instance.py --instance-id + +Output JSON: +{ + "success": true, + "platform": "bm", + "instance_id": "", + "public_ip": "...", + "private_ip": "...", + "state": "running", + "ssh_user": "root", + "key_file": "" +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Describe Carbide bare-metal instance") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "")) + parser.add_argument("--instance-id", default="") + args = parser.parse_args() + + if not args.instance_id: + state = load_state() + args.instance_id = state.get("instance_id", "") + + if not args.instance_id: + print(json.dumps({ + "success": False, + "platform": "bm", + "error": "instance-id is required", + }, indent=2)) + return 1 + + result: dict[str, Any] = { + "success": False, + "platform": "bm", + "instance_id": args.instance_id, + } + + try: + resp = run_carbide("instance", "get", "--id", args.instance_id) + + status = resp.get("status", resp.get("state", "")) + result["public_ip"] = resp.get("public_ip", resp.get("ip_address", "")) + result["private_ip"] = resp.get("private_ip", resp.get("internal_ip", "")) + result["state"] = "running" if status.lower() in ("running", "active") else status + result["ssh_user"] = "root" + result["key_file"] = "" + result["success"] = result["state"] == "running" + + if not result["success"]: + result["error"] = f"Instance {args.instance_id} is {result['state']}, expected running" + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/bm/launch_instance.py b/isvctl/configs/stubs/carbide/bm/launch_instance.py new file mode 100644 index 00000000..0118b1de --- /dev/null +++ b/isvctl/configs/stubs/carbide/bm/launch_instance.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Launch a Carbide bare-metal GPU instance. + +Uses ``carbidecli instance create`` to provision a bare-metal instance +with the specified OS and instance type. Waits for the instance to +reach running state. + +Usage: + python launch_instance.py --site-id --os-id --instance-type + +Output JSON: +{ + "success": true, + "platform": "bm", + "instance_id": "", + "public_ip": "...", + "private_ip": "...", + "state": "running", + "ssh_user": "root", + "key_file": "", + "vpc_id": "" +} +""" + +import argparse +import json +import os +import sys +import time +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide, save_state + + +def wait_for_instance(instance_id: str, timeout: int = 600) -> dict[str, Any]: + """Poll instance status until running or timeout.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + resp = run_carbide("instance", "get", "--id", instance_id) + state = resp.get("status", resp.get("state", "")) + if state.lower() in ("running", "active"): + return resp + print(f"Instance {instance_id} state: {state}, waiting...", file=sys.stderr) + time.sleep(15) + raise RuntimeError(f"Instance {instance_id} did not reach running state within {timeout}s") + + +def main() -> int: + parser = argparse.ArgumentParser(description="Launch Carbide bare-metal instance") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "")) + parser.add_argument("--site-id", default=os.environ.get("CARBIDE_SITE_ID", "")) + parser.add_argument("--os-id", default=os.environ.get("CARBIDE_OS_ID", "")) + parser.add_argument("--instance-type", default=os.environ.get("CARBIDE_INSTANCE_TYPE", "")) + parser.add_argument("--name", default="ncp-bm-test-gpu") + parser.add_argument("--instance-id", default=os.environ.get("CARBIDE_INSTANCE_ID", "")) + parser.add_argument("--vpc-id", default=os.environ.get("CARBIDE_VPC_ID", "")) + args = parser.parse_args() + + result: dict[str, Any] = { + "success": False, + "platform": "bm", + } + + try: + state = load_state() + + if args.instance_id: + # Use pre-existing instance + instance_data = run_carbide("instance", "get", "--id", args.instance_id) + instance_id = instance_data.get("id", args.instance_id) + state["instance_created"] = False + else: + # Create new instance + if not args.site_id or not args.os_id or not args.instance_type: + result["error"] = "site-id, os-id, and instance-type required when not using pre-existing instance" + print(json.dumps(result, indent=2)) + return 1 + + instance_name = f"{args.name}-{int(time.time())}" + create_args = [ + "instance", "create", + "--name", instance_name, + "--operating-system-id", args.os_id, + "--site-id", args.site_id, + "--instance-type", args.instance_type, + ] + if args.vpc_id: + create_args += ["--vpc-id", args.vpc_id] + + resp = run_carbide(*create_args) + instance_id = resp.get("id", resp.get("instance_id", "")) + state["instance_created"] = True + + # Wait for instance to be running + instance_data = wait_for_instance(instance_id) + + result["instance_id"] = instance_id + result["public_ip"] = instance_data.get("public_ip", instance_data.get("ip_address", "")) + result["private_ip"] = instance_data.get("private_ip", instance_data.get("internal_ip", "")) + result["state"] = "running" + result["ssh_user"] = "root" + result["key_file"] = "" + result["vpc_id"] = instance_data.get("vpc_id", instance_data.get("network_id", "")) + result["success"] = True + + state["instance_id"] = instance_id + state["site_id"] = args.site_id + save_state(state) + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/bm/list_instances.py b/isvctl/configs/stubs/carbide/bm/list_instances.py new file mode 100644 index 00000000..3591becb --- /dev/null +++ b/isvctl/configs/stubs/carbide/bm/list_instances.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""List Carbide instances and verify the target instance is present. + +Uses ``carbidecli instance list`` to enumerate instances at the site, +then checks whether the target instance ID appears in the list. + +Usage: + python list_instances.py --site-id --instance-id + +Output JSON: +{ + "success": true, + "platform": "bm", + "instances": [...], + "count": N, + "found_target": true, + "target_instance": "" +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="List Carbide instances") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "")) + parser.add_argument("--site-id", default=os.environ.get("CARBIDE_SITE_ID", "")) + parser.add_argument("--instance-id", default="") + args = parser.parse_args() + + if not args.instance_id: + state = load_state() + args.instance_id = state.get("instance_id", "") + + result: dict[str, Any] = { + "success": False, + "platform": "bm", + "instances": [], + "count": 0, + "found_target": False, + "target_instance": args.instance_id, + } + + try: + list_args = ["instance", "list"] + if args.site_id: + list_args.extend(["--site-id", args.site_id]) + + resp = run_carbide(*list_args) + + # Response may be a list directly or wrapped in a key + instances = resp if isinstance(resp, list) else resp.get("instances", resp.get("items", [])) + result["instances"] = instances + result["count"] = len(instances) + + # Check if target instance is in the list + if args.instance_id: + for inst in instances: + inst_id = inst.get("id", inst.get("instance_id", "")) + if inst_id == args.instance_id: + result["found_target"] = True + break + + result["success"] = True + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/bm/reboot_instance.py b/isvctl/configs/stubs/carbide/bm/reboot_instance.py new file mode 100644 index 00000000..3da6c2d1 --- /dev/null +++ b/isvctl/configs/stubs/carbide/bm/reboot_instance.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Reboot a Carbide bare-metal instance and wait for it to come back. + +Uses ``carbidecli instance reboot`` then polls until the instance +returns to running state and SSH is reachable. + +Usage: + python reboot_instance.py --instance-id --public-ip + +Output JSON: +{ + "success": true, + "platform": "bm", + "instance_id": "", + "reboot_initiated": true, + "state": "running", + "ssh_ready": true, + "uptime_seconds": 120 +} +""" + +import argparse +import json +import os +import socket +import sys +import time +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def check_ssh(host: str, port: int = 22, timeout: int = 5) -> bool: + """Check if SSH port is reachable.""" + try: + with socket.create_connection((host, port), timeout=timeout): + return True + except (OSError, socket.timeout): + return False + + +def wait_for_ready(instance_id: str, public_ip: str, timeout: int = 600) -> dict[str, Any]: + """Wait for instance to be running and SSH-reachable after reboot.""" + deadline = time.monotonic() + timeout + reboot_time = time.monotonic() + + # Wait for instance to return to running state + while time.monotonic() < deadline: + try: + resp = run_carbide("instance", "get", "--id", instance_id) + state = resp.get("status", resp.get("state", "")) + if state.lower() in ("running", "active"): + break + except Exception: + pass + time.sleep(10) + + # Wait for SSH to be reachable + ssh_ready = False + while time.monotonic() < deadline: + if check_ssh(public_ip): + ssh_ready = True + break + time.sleep(10) + + uptime = int(time.monotonic() - reboot_time) + return {"ssh_ready": ssh_ready, "uptime_seconds": uptime} + + +def main() -> int: + parser = argparse.ArgumentParser(description="Reboot Carbide bare-metal instance") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "")) + parser.add_argument("--instance-id", default="") + parser.add_argument("--public-ip", default="") + args = parser.parse_args() + + if not args.instance_id: + state = load_state() + args.instance_id = state.get("instance_id", "") + + if not args.instance_id: + print(json.dumps({ + "success": False, + "platform": "bm", + "error": "instance-id is required", + }, indent=2)) + return 1 + + result: dict[str, Any] = { + "success": False, + "platform": "bm", + "instance_id": args.instance_id, + "reboot_initiated": False, + } + + try: + run_carbide("instance", "reboot", "--id", args.instance_id) + result["reboot_initiated"] = True + + if args.public_ip: + ready = wait_for_ready(args.instance_id, args.public_ip) + result["ssh_ready"] = ready["ssh_ready"] + result["uptime_seconds"] = ready["uptime_seconds"] + else: + result["ssh_ready"] = False + result["uptime_seconds"] = 0 + + result["state"] = "running" + result["success"] = True + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/bm/reinstall_instance.py b/isvctl/configs/stubs/carbide/bm/reinstall_instance.py new file mode 100644 index 00000000..2a8e2dd7 --- /dev/null +++ b/isvctl/configs/stubs/carbide/bm/reinstall_instance.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Reinstall OS on a Carbide bare-metal instance (not supported). + +Carbide does not currently support in-place OS reinstall. This stub +outputs a success result with a skip note so the template validation +passes gracefully. + +Usage: + python reinstall_instance.py + +Output JSON: +{ + "success": true, + "platform": "bm", + "skipped": true, + "message": "Reinstall not supported by Carbide" +} +""" + +import argparse +import json +import os +import sys + + +def main() -> int: + parser = argparse.ArgumentParser(description="Reinstall Carbide BM instance (not supported)") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "")) + parser.parse_args() + + result = { + "success": True, + "platform": "bm", + "skipped": True, + "message": "Reinstall not supported by Carbide", + } + + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/bm/teardown.py b/isvctl/configs/stubs/carbide/bm/teardown.py new file mode 100644 index 00000000..c5642370 --- /dev/null +++ b/isvctl/configs/stubs/carbide/bm/teardown.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Delete a Carbide bare-metal instance. + +Uses ``carbidecli instance delete`` to terminate the instance. + +Usage: + python teardown.py --instance-id + +Output JSON: +{ + "success": true, + "platform": "bm", + "resources_deleted": [...] +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide, save_state + + +def main() -> int: + parser = argparse.ArgumentParser(description="Teardown Carbide bare-metal instance") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "")) + parser.add_argument("--instance-id", default="") + args = parser.parse_args() + + state = load_state() + if not args.instance_id: + args.instance_id = state.get("instance_id", "") + + result: dict[str, Any] = { + "success": False, + "platform": "bm", + "resources_deleted": [], + } + + if not args.instance_id: + result["success"] = True + result["message"] = "No instance to delete" + print(json.dumps(result, indent=2)) + return 0 + + if not state.get("instance_created", True): + result["success"] = True + result["skipped"] = True + result["message"] = f"Instance {args.instance_id} is pre-existing, not deleting" + print(json.dumps(result, indent=2)) + return 0 + + try: + run_carbide("instance", "delete", "--id", args.instance_id) + result["resources_deleted"].append(f"instance:{args.instance_id}") + result["success"] = True + + save_state({}) + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/bm/verify_terminated.py b/isvctl/configs/stubs/carbide/bm/verify_terminated.py new file mode 100644 index 00000000..c5a36e65 --- /dev/null +++ b/isvctl/configs/stubs/carbide/bm/verify_terminated.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Verify a Carbide bare-metal instance has been terminated. + +Post-teardown sanitization check: uses ``carbidecli instance get`` +to confirm the instance no longer exists or is in a terminated state. + +Usage: + python verify_terminated.py --instance-id + +Output JSON: +{ + "success": true, + "platform": "bm", + "checks": {"instance_terminated": true} +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Verify Carbide instance terminated") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "")) + parser.add_argument("--instance-id", default="") + args = parser.parse_args() + + if not args.instance_id: + print(json.dumps({ + "success": False, + "platform": "bm", + "error": "instance-id is required", + }, indent=2)) + return 1 + + result: dict[str, Any] = { + "success": False, + "platform": "bm", + "checks": {"instance_terminated": False}, + } + + try: + resp = run_carbide("instance", "get", "--id", args.instance_id) + # If we get a response, check that the state is terminated/deleted + state = resp.get("status", resp.get("state", "")) + if state.lower() in ("terminated", "deleted", "destroying", "destroyed"): + result["checks"]["instance_terminated"] = True + result["success"] = True + else: + result["error"] = f"Instance {args.instance_id} still in state: {state}" + except RuntimeError: + # Command failed — instance no longer exists, which means it was deleted + result["checks"]["instance_terminated"] = True + result["success"] = True + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/common/__init__.py b/isvctl/configs/stubs/carbide/common/__init__.py new file mode 100644 index 00000000..442bd7fe --- /dev/null +++ b/isvctl/configs/stubs/carbide/common/__init__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. diff --git a/isvctl/configs/stubs/carbide/common/carbide.py b/isvctl/configs/stubs/carbide/common/carbide.py new file mode 100644 index 00000000..2fd0f17c --- /dev/null +++ b/isvctl/configs/stubs/carbide/common/carbide.py @@ -0,0 +1,245 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Shared helpers for Carbide CLI (carbidecli) stub scripts. + +Provides: + - run_carbide(): Execute carbidecli commands with JSON output parsing + - timed_call(): Same as run_carbide but also returns latency + - load_state() / save_state(): Persist data between steps via JSON file + - get_scopes(): Parse TARGET_SCOPES env var into structured dict + - check_scopes(): Verify required scopes are present + +Environment: + carbidecli handles authentication via its own config (~/.carbide/config.yaml) + or environment variables (CARBIDE_TOKEN, CARBIDE_API_KEY, CARBIDE_ORG, etc.). + TARGET_SCOPES contains the list of granted API scopes. +""" + +import json +import os +import subprocess +import time +from pathlib import Path +from typing import Any + + +DEFAULT_STATE_FILE = "/tmp/ncp-carbide-state.json" + + +def run_carbide(*args: str, timeout: int = 120) -> dict[str, Any]: + """Run a carbidecli command and return parsed JSON output. + + Args: + *args: Command arguments (e.g., "tenant", "get") + timeout: Command timeout in seconds + + Returns: + Parsed JSON output from carbidecli + + Raises: + RuntimeError: If the command fails or returns non-JSON output + """ + cmd = ["carbidecli", "-o", "json"] + list(args) + result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + + if result.returncode != 0: + raise RuntimeError(f"carbidecli {' '.join(args)} failed: {result.stderr.strip()}") + + try: + return json.loads(result.stdout) + except json.JSONDecodeError: + raise RuntimeError(f"carbidecli returned non-JSON output: {result.stdout[:500]}") + + +def timed_call(*args: str, timeout: int = 120) -> tuple[dict[str, Any], float]: + """Run a carbidecli command and return (result, latency_seconds).""" + start = time.monotonic() + data = run_carbide(*args, timeout=timeout) + elapsed = time.monotonic() - start + return data, elapsed + + +def get_scopes() -> dict[str, set[str]]: + """Parse TARGET_SCOPES env var into {resource: {read, write}} dict. + + Example: + TARGET_SCOPES="forge:read_vpc forge:write_vpc forge:read_tenant" + → {"vpc": {"read", "write"}, "tenant": {"read"}} + """ + scopes_str = os.environ.get("TARGET_SCOPES", "") + result: dict[str, set[str]] = {} + for scope in scopes_str.split(): + # Format: forge:read_resource or forge:write_resource + scope = scope.removeprefix("forge:") + if scope.startswith("read_"): + resource = scope[5:] + result.setdefault(resource, set()).add("read") + elif scope.startswith("write_"): + resource = scope[6:] + result.setdefault(resource, set()).add("write") + return result + + +def check_scopes( + required: dict[str, list[str]], +) -> tuple[bool, list[str], list[str]]: + """Check if required scopes are present. + + Args: + required: {resource: [operations]} e.g., {"vpc": ["read", "write"]} + + Returns: + (all_present, granted_list, missing_list) + """ + scopes = get_scopes() + granted = [] + missing = [] + for resource, operations in required.items(): + resource_scopes = scopes.get(resource, set()) + for op in operations: + scope_str = f"{op}_{resource}" + if op in resource_scopes: + granted.append(scope_str) + else: + missing.append(scope_str) + return len(missing) == 0, granted, missing + + +# ========================================================================= +# Carbide API resource definitions +# ========================================================================= + +# All resources from the Carbide OpenAPI spec (bare-metal-manager-rest) +# with their supported operations. Derived from openapi/spec.yaml. +# Scope names use underscores (matching TARGET_SCOPES format). +CARBIDE_API_RESOURCES: dict[str, dict[str, list[str]]] = { + # Core infrastructure + "site": {"ops": ["create", "list", "get", "update", "delete"], "scope": "site"}, + "vpc": {"ops": ["create", "list", "get", "update", "delete"], "scope": "vpc"}, + "vpc-prefix": {"ops": ["create", "list", "get", "update", "delete"], "scope": "vpc_prefix"}, + "subnet": {"ops": ["create", "list", "get", "update", "delete"], "scope": "subnet"}, + "network-security-group": {"ops": ["create", "list", "get", "update", "delete"], "scope": "network_security_group"}, + "ipblock": {"ops": ["create", "list", "get", "update", "delete"], "scope": "ip_block"}, + "allocation": {"ops": ["create", "list", "get", "update", "delete"], "scope": "allocation"}, + # Compute + "instance": {"ops": ["create", "list", "get", "update", "delete"], "scope": "instance"}, + "instance-type": {"ops": ["create", "list", "get", "update", "delete"], "scope": "instance_type"}, + "machine": {"ops": ["list", "get", "update", "delete"], "scope": "machine"}, + "expected-machine": {"ops": ["create", "list", "get", "update", "delete"], "scope": "expected_machine"}, + "operating-system": {"ops": ["create", "list", "get", "update", "delete"], "scope": "operation_system"}, + # Networking / fabric + "infiniband-partition": {"ops": ["create", "list", "get", "update", "delete"], "scope": "infini_band_partition"}, + "nvlink-logical-partition": {"ops": ["create", "list", "get", "update", "delete"], "scope": None}, + "nvlink-interface": {"ops": ["list"], "scope": None}, + "dpu-extension-service": {"ops": ["create", "list", "get", "update", "delete"], "scope": None}, + # Identity / access + "sshkeygroup": {"ops": ["create", "list", "get", "update", "delete"], "scope": "ssh_key_group"}, + "sshkey": {"ops": ["create", "list", "get", "update", "delete"], "scope": "ssh_key"}, + "tenant": {"ops": ["list", "get"], "scope": "tenant"}, + # Hardware topology + "rack": {"ops": ["list", "get"], "scope": None}, + "tray": {"ops": ["list", "get"], "scope": None}, + "sku": {"ops": ["list", "get"], "scope": None}, + "machine-capability": {"ops": ["list"], "scope": None}, + # Observability + "audit": {"ops": ["list", "get"], "scope": "audit"}, + "metadata": {"ops": ["list"], "scope": None}, +} + +# Resource → env var for pre-existing resources. +# When set, the template only needs "read" (not "write") for that resource. +PREEXISTING_ENV_VARS: dict[str, str] = { + "vpc": "CARBIDE_VPC_ID", + "vpc_prefix": "CARBIDE_VPC_PREFIX_ID", + "subnet": "CARBIDE_SUBNET_ID", + "ssh_key_group": "CARBIDE_SSH_KEY_GROUP_ID", + "operation_system": "CARBIDE_OS_ID", + "instance_type": "CARBIDE_INSTANCE_TYPE", + "instance": "CARBIDE_INSTANCE_ID", +} + +# Minimum scopes each template requires to run (when creating all resources). +TEMPLATE_REQUIRED_SCOPES: dict[str, dict[str, list[str]]] = { + "iam": { + "tenant": ["read"], + "site": ["read"], + "ssh_key_group": ["read", "write"], + "ssh_key": ["read", "write"], + }, + "control-plane": { + "tenant": ["read"], + "site": ["read"], + "ssh_key_group": ["read", "write"], + "ssh_key": ["read", "write"], + "vpc": ["read", "write"], + }, + "network": { + "vpc": ["read", "write"], + "vpc_prefix": ["read", "write"], + "subnet": ["read", "write"], + "network_security_group": ["read", "write"], + }, + "image-registry": { + "operation_system": ["read", "write"], + "instance_type": ["read"], + "instance": ["read", "write"], + }, + "bm": { + "instance": ["read", "write"], + "instance_type": ["read"], + "operation_system": ["read"], + "vpc": ["read"], + }, +} + +# Keep backward-compatible alias +TEMPLATE_SCOPES = TEMPLATE_REQUIRED_SCOPES + + +def effective_scopes_for_template(template_name: str) -> dict[str, list[str]]: + """Calculate effective required scopes accounting for pre-existing resources. + + When a CARBIDE_*_ID env var is set for a resource, write permission + is not needed — only read. This lets users run templates in restricted + environments where infrastructure is shared or pre-provisioned. + + Args: + template_name: Template name (e.g., "control-plane", "network") + + Returns: + {resource: [operations]} with write removed where pre-existing + """ + base = TEMPLATE_REQUIRED_SCOPES.get(template_name, {}) + effective: dict[str, list[str]] = {} + + for resource, operations in base.items(): + env_var = PREEXISTING_ENV_VARS.get(resource) + if env_var and os.environ.get(env_var): + # Pre-existing: only need read + effective[resource] = ["read"] + else: + effective[resource] = list(operations) + + return effective + + +def load_state(state_file: str | None = None) -> dict[str, Any]: + """Load persisted state from a JSON file.""" + path = Path(state_file or os.environ.get("CARBIDE_STATE_FILE", DEFAULT_STATE_FILE)) + if path.exists(): + return json.loads(path.read_text()) + return {} + + +def save_state(state: dict[str, Any], state_file: str | None = None) -> None: + """Save state to a JSON file for use by subsequent steps.""" + path = Path(state_file or os.environ.get("CARBIDE_STATE_FILE", DEFAULT_STATE_FILE)) + path.write_text(json.dumps(state, indent=2)) diff --git a/isvctl/configs/stubs/carbide/common/resources.py b/isvctl/configs/stubs/carbide/common/resources.py new file mode 100644 index 00000000..3d408e70 --- /dev/null +++ b/isvctl/configs/stubs/carbide/common/resources.py @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Generic CRUD operations for all Carbide API resources. + +Provides create/get/list/delete for every resource type that carbidecli +supports. Stubs should use these instead of calling run_carbide directly +for standard CRUD operations. + +Usage: + from common.resources import CarbideResource + + vpc = CarbideResource("vpc") + result = vpc.create(name="my-vpc", site_id="...") + vpc.get(result["id"]) + vpc.list() + vpc.delete(result["id"]) +""" + +from typing import Any + +from .carbide import run_carbide + + +class CarbideResource: + """Generic CRUD wrapper for a Carbide API resource. + + Args: + resource_type: The carbidecli resource name (e.g., "vpc", "subnet", + "operating-system", "ssh-key-group") + """ + + def __init__(self, resource_type: str) -> None: + self.resource_type = resource_type + + def create(self, **kwargs: str) -> dict[str, Any]: + """Create a resource. Kwargs are passed as --key value args.""" + args = [self.resource_type, "create"] + for key, value in kwargs.items(): + if value: + args.extend([f"--{key.replace('_', '-')}", value]) + return run_carbide(*args) + + def get(self, resource_id: str) -> dict[str, Any]: + """Get a resource by ID.""" + return run_carbide(self.resource_type, "get", resource_id) + + def list(self, **kwargs: str) -> list[dict[str, Any]] | dict[str, Any]: + """List resources. Kwargs are passed as filter args.""" + args = [self.resource_type, "list"] + for key, value in kwargs.items(): + if value: + args.extend([f"--{key.replace('_', '-')}", value]) + result = run_carbide(*args) + return result if isinstance(result, list) else result + + def delete(self, resource_id: str) -> bool: + """Delete a resource by ID. Returns True on success or already-deleted.""" + try: + run_carbide(self.resource_type, "delete", resource_id) + return True + except RuntimeError as e: + if "not found" in str(e).lower() or "404" in str(e): + return True + raise + + def update(self, resource_id: str, **kwargs: str) -> dict[str, Any]: + """Update a resource. Kwargs are passed as --key value args.""" + args = [self.resource_type, "update", resource_id] + for key, value in kwargs.items(): + if value: + args.extend([f"--{key.replace('_', '-')}", value]) + return run_carbide(*args) + + +# Pre-configured resource instances for all Carbide API resources. +# Names match the carbidecli subcommand names. + +# Core infrastructure +site = CarbideResource("site") +vpc = CarbideResource("vpc") +vpc_prefix = CarbideResource("vpc-prefix") +subnet = CarbideResource("subnet") +nsg = CarbideResource("network-security-group") +ipblock = CarbideResource("ipblock") +allocation = CarbideResource("allocation") + +# Compute +instance = CarbideResource("instance") +instance_type = CarbideResource("instance-type") +machine = CarbideResource("machine") +expected_machine = CarbideResource("expected-machine") +operating_system = CarbideResource("operating-system") + +# Networking / fabric +infiniband_partition = CarbideResource("infiniband-partition") +nvlink_logical_partition = CarbideResource("nvlink-logical-partition") +nvlink_interface = CarbideResource("nvlink-interface") +dpu_extension_service = CarbideResource("dpu-extension-service") + +# Identity / access +ssh_key_group = CarbideResource("sshkeygroup") +ssh_key = CarbideResource("sshkey") +tenant = CarbideResource("tenant") + +# Hardware topology +rack = CarbideResource("rack") +tray = CarbideResource("tray") +sku = CarbideResource("sku") + +# Observability +audit = CarbideResource("audit") diff --git a/isvctl/configs/stubs/carbide/control-plane/check_api.py b/isvctl/configs/stubs/carbide/control-plane/check_api.py new file mode 100644 index 00000000..ca92c5b5 --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/check_api.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Check Carbide API connectivity and health. + +Verifies the Carbide control plane is reachable by running +``carbidecli tenant get`` and ``carbidecli site list``. + +Usage: + python check_api.py --region us-west-2 + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "account_id": "", + "tests": { + "tenant": {"passed": true, "latency_ms": 123}, + "sites": {"passed": true, "latency_ms": 89} + } +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import timed_call + + +def main() -> int: + parser = argparse.ArgumentParser(description="Check Carbide API health") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--services", default="tenant,sites", help="Comma-separated checks") + args = parser.parse_args() + + result: dict[str, Any] = { + "success": False, + "platform": "control_plane", + "tests": {}, + } + + try: + # Test tenant get + tenant_data, tenant_latency = timed_call("tenant", "get") + result["tests"]["tenant"] = { + "passed": True, + "latency_ms": round(tenant_latency * 1000, 2), + } + # Extract tenant/account ID from response + result["account_id"] = tenant_data.get("id", tenant_data.get("tenant_id", "")) + + # Test site list + sites_data, sites_latency = timed_call("site", "list") + result["tests"]["sites"] = { + "passed": True, + "latency_ms": round(sites_latency * 1000, 2), + } + + passed = sum(1 for t in result["tests"].values() if t.get("passed", False)) + total = len(result["tests"]) + result["summary"] = f"{passed}/{total} checks passed" + result["success"] = True + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/control-plane/create_access_key.py b/isvctl/configs/stubs/carbide/control-plane/create_access_key.py new file mode 100644 index 00000000..4718fbfb --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/create_access_key.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Create SSH key group and SSH key in Carbide. + +Maps the template's "access key" concept to Carbide's SSH key model: +an SSH key group is created first, then an SSH key within it. + +Usage: + python create_access_key.py --username ncp-validation + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "username": "ncp-validation", + "user_id": "", + "access_key_id": "", + "secret_access_key": "" +} +""" + +import argparse +import json +import os +import sys +import time +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide, save_state + + +def main() -> int: + parser = argparse.ArgumentParser(description="Create Carbide SSH key group and key") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--username-prefix", default="ncp-validation") + args = parser.parse_args() + + suffix = int(time.time()) + group_name = f"{args.username_prefix}-group-{suffix}" + key_name = f"{args.username_prefix}-key-{suffix}" + + result: dict[str, Any] = { + "success": False, + "platform": "control_plane", + "username": args.username_prefix, + } + + existing_group_id = os.environ.get("CARBIDE_SSH_KEY_GROUP_ID", "") + + try: + state = load_state() + + # SSH key group: reuse or create + if existing_group_id: + group_resp = run_carbide("ssh-key-group", "get", existing_group_id) + group_id = group_resp.get("id", existing_group_id) + group_name = group_resp.get("name", existing_group_id) + state["ssh_key_group_created"] = False + else: + group_resp = run_carbide("ssh-key-group", "create", "--name", group_name) + group_id = group_resp.get("id", group_resp.get("ssh_key_group_id", "")) + state["ssh_key_group_created"] = True + + result["user_id"] = group_id + + # SSH key: always create (it's the test artifact) + key_resp = run_carbide( + "ssh-key", "create", + "--name", key_name, + "--ssh-key-group-id", group_id, + ) + key_id = key_resp.get("id", key_resp.get("ssh_key_id", "")) + public_key = key_resp.get("public_key", key_resp.get("key", "")) + + result["access_key_id"] = key_id + result["secret_access_key"] = public_key + result["success"] = True + + state["ssh_key_group_id"] = group_id + state["ssh_key_group_name"] = group_name + state["ssh_key_id"] = key_id + state["ssh_key_name"] = key_name + state["username"] = args.username_prefix + save_state(state) + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/control-plane/create_tenant.py b/isvctl/configs/stubs/carbide/control-plane/create_tenant.py new file mode 100644 index 00000000..32262809 --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/create_tenant.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Create or reuse a VPC in Carbide (maps to template's "tenant" concept). + +If ``CARBIDE_VPC_ID`` is set, uses the pre-existing VPC instead of creating +a new one. Only VPCs created by this script are deleted during teardown. + +Requires a site ID when creating, provided via ``--site-id`` or the +``CARBIDE_SITE_ID`` environment variable. + +Usage: + python create_tenant.py --site-id + CARBIDE_VPC_ID= python create_tenant.py + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "tenant_name": "ncp-vpc-", + "tenant_id": "", + "description": "NCP validation VPC" +} +""" + +import argparse +import json +import os +import sys +import time +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide, save_state + + +def main() -> int: + parser = argparse.ArgumentParser(description="Create Carbide VPC") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--name-prefix", default="ncp-vpc") + parser.add_argument("--site-id", default=os.environ.get("CARBIDE_SITE_ID", "")) + parser.add_argument("--vpc-id", default=os.environ.get("CARBIDE_VPC_ID", "")) + args = parser.parse_args() + + result: dict[str, Any] = { + "success": False, + "platform": "control_plane", + } + + try: + state = load_state() + + if args.vpc_id: + # Use pre-existing VPC + resp = run_carbide("vpc", "get", args.vpc_id) + vpc_id = resp.get("id", args.vpc_id) + vpc_name = resp.get("name", args.vpc_id) + state["vpc_created"] = False + else: + # Create new VPC + if not args.site_id: + result["error"] = "site-id is required when not using pre-existing VPC" + print(json.dumps(result, indent=2)) + return 1 + + vpc_name = f"{args.name_prefix}-{int(time.time())}" + resp = run_carbide( + "vpc", "create", + "--name", vpc_name, + "--description", "NCP validation VPC", + "--site-id", args.site_id, + ) + vpc_id = resp.get("id", resp.get("vpc_id", "")) + state["vpc_created"] = True + + result["tenant_name"] = vpc_name + result["tenant_id"] = vpc_id + result["description"] = "NCP validation VPC" + result["success"] = True + + state["vpc_id"] = vpc_id + state["vpc_name"] = vpc_name + state["site_id"] = args.site_id + save_state(state) + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/control-plane/delete_access_key.py b/isvctl/configs/stubs/carbide/control-plane/delete_access_key.py new file mode 100644 index 00000000..b58a327d --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/delete_access_key.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Delete SSH key group in Carbide (teardown for access key lifecycle). + +The SSH key itself was already deleted by ``disable_access_key``; +this step removes the parent SSH key group. + +Usage: + python delete_access_key.py --username ncp-validation --access-key-id + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "resources_deleted": ["ssh-key-group/"] +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Delete Carbide SSH key group") + parser.add_argument("--username", default="") + parser.add_argument("--access-key-id", default="") + parser.add_argument("--skip-destroy", action="store_true") + args = parser.parse_args() + + result: dict[str, Any] = {"success": False, "platform": "control_plane"} + + if args.skip_destroy: + result["success"] = True + result["skipped"] = True + print(json.dumps(result, indent=2)) + return 0 + + state = load_state() + group_id = state.get("ssh_key_group_id", "") + + if not state.get("ssh_key_group_created", True): + result["success"] = True + result["skipped"] = True + result["message"] = f"SSH key group {group_id} is pre-existing, not deleting" + result["resources_deleted"] = [] + print(json.dumps(result, indent=2)) + return 0 + + try: + run_carbide("ssh-key-group", "delete", "--id", group_id) + result["resources_deleted"] = [f"ssh-key-group/{group_id}"] + result["success"] = True + except RuntimeError as e: + if "not found" in str(e).lower() or "404" in str(e): + result["resources_deleted"] = [f"ssh-key-group/{group_id}"] + result["success"] = True + result["already_deleted"] = True + else: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/control-plane/delete_tenant.py b/isvctl/configs/stubs/carbide/control-plane/delete_tenant.py new file mode 100644 index 00000000..57d7ffb8 --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/delete_tenant.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Delete VPC in Carbide (teardown for tenant lifecycle). + +Usage: + python delete_tenant.py --group-name ncp-vpc-1234567890 + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "resources_deleted": ["vpc/"] +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Delete Carbide VPC") + parser.add_argument("--group-name", default="", help="VPC name (unused, ID from state)") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--skip-destroy", action="store_true") + args = parser.parse_args() + + result: dict[str, Any] = {"success": False, "platform": "control_plane"} + + if args.skip_destroy: + result["success"] = True + result["skipped"] = True + print(json.dumps(result, indent=2)) + return 0 + + state = load_state() + vpc_id = state.get("vpc_id", "") + + if not state.get("vpc_created", True): + # Pre-existing VPC — don't delete + result["success"] = True + result["skipped"] = True + result["message"] = f"VPC {vpc_id} is pre-existing, not deleting" + result["resources_deleted"] = [] + print(json.dumps(result, indent=2)) + return 0 + + try: + run_carbide("vpc", "delete", "--id", vpc_id) + result["resources_deleted"] = [f"vpc/{vpc_id}"] + result["success"] = True + except RuntimeError as e: + if "not found" in str(e).lower() or "404" in str(e): + result["resources_deleted"] = [f"vpc/{vpc_id}"] + result["success"] = True + result["already_deleted"] = True + else: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/control-plane/disable_access_key.py b/isvctl/configs/stubs/carbide/control-plane/disable_access_key.py new file mode 100644 index 00000000..72fdfbf8 --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/disable_access_key.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Disable (delete) the SSH key in Carbide. + +Carbide does not support disabling SSH keys, so this script deletes +the key instead. The output uses the template's expected field names +(``status: Inactive``) for compatibility. + +Usage: + python disable_access_key.py --username ncp-validation --access-key-id + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "access_key_id": "", + "status": "Inactive" +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Disable (delete) Carbide SSH key") + parser.add_argument("--username", default="") + parser.add_argument("--access-key-id", default="") + args = parser.parse_args() + + state = load_state() + key_id = args.access_key_id or state.get("ssh_key_id", "") + + result: dict[str, Any] = { + "success": False, + "platform": "control_plane", + "access_key_id": key_id, + } + + try: + run_carbide("ssh-key", "delete", "--id", key_id) + result["status"] = "Inactive" + result["success"] = True + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/control-plane/get_tenant.py b/isvctl/configs/stubs/carbide/control-plane/get_tenant.py new file mode 100644 index 00000000..3c33bee6 --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/get_tenant.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Get VPC details from Carbide (maps to template's "get tenant" step). + +Usage: + python get_tenant.py --group-name ncp-vpc-1234567890 + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "tenant_name": "ncp-vpc-1234567890", + "tenant_id": "", + "description": "NCP validation VPC" +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Get Carbide VPC details") + parser.add_argument("--group-name", default="", help="VPC name (or loaded from state)") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + args = parser.parse_args() + + state = load_state() + vpc_id = state.get("vpc_id", "") + vpc_name = args.group_name or state.get("vpc_name", "") + + result: dict[str, Any] = { + "success": False, + "platform": "control_plane", + "tenant_name": vpc_name, + } + + try: + resp = run_carbide("vpc", "get", "--id", vpc_id) + + result["tenant_id"] = resp.get("id", resp.get("vpc_id", "")) + result["tenant_name"] = resp.get("name", vpc_name) + result["description"] = resp.get("description", "NCP validation VPC") + result["success"] = True + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/control-plane/list_tenants.py b/isvctl/configs/stubs/carbide/control-plane/list_tenants.py new file mode 100644 index 00000000..d0fdacd2 --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/list_tenants.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""List VPCs in Carbide (maps to template's "list tenants" step). + +Usage: + python list_tenants.py --target-group ncp-vpc-1234567890 + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "tenants": [{"tenant_name": "...", "tenant_id": "..."}], + "count": 1, + "found_target": true, + "target_tenant": "ncp-vpc-1234567890" +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="List Carbide VPCs") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--target-group", default="", help="VPC name to verify exists") + args = parser.parse_args() + + state = load_state() + target = args.target_group or state.get("vpc_name", "") + + result: dict[str, Any] = { + "success": False, + "platform": "control_plane", + "tenants": [], + } + + try: + resp = run_carbide("vpc", "list") + vpcs = resp if isinstance(resp, list) else resp.get("items", []) + + for vpc in vpcs: + result["tenants"].append({ + "tenant_name": vpc.get("name", ""), + "tenant_id": vpc.get("id", vpc.get("vpc_id", "")), + }) + + result["count"] = len(result["tenants"]) + + if target: + result["target_tenant"] = target + result["found_target"] = any( + t["tenant_name"] == target for t in result["tenants"] + ) + + result["success"] = True + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/control-plane/test_access_key.py b/isvctl/configs/stubs/carbide/control-plane/test_access_key.py new file mode 100644 index 00000000..50e4dcdf --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/test_access_key.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Test that the created SSH key is visible in Carbide. + +Lists SSH keys and verifies the key created by ``create_access_key`` +appears in the listing. + +Usage: + python test_access_key.py --access-key-id + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "authenticated": true, + "identity_id": "", + "account_id": "" +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Test Carbide SSH key visibility") + parser.add_argument("--access-key-id", default="") + parser.add_argument("--secret-access-key", default="") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--wait", type=int, default=0, help="Seconds to wait (unused)") + parser.add_argument("--retries", type=int, default=3, help="Number of retry attempts") + args = parser.parse_args() + + state = load_state() + key_id = args.access_key_id or state.get("ssh_key_id", "") + group_id = state.get("ssh_key_group_id", "") + + result: dict[str, Any] = { + "success": False, + "platform": "control_plane", + "authenticated": False, + } + + try: + # List SSH keys and look for our key + keys_resp = run_carbide("ssh-key", "list") + keys = keys_resp if isinstance(keys_resp, list) else keys_resp.get("items", []) + + found = any( + k.get("id", k.get("ssh_key_id", "")) == key_id + for k in keys + ) + + if found: + result["authenticated"] = True + result["identity_id"] = key_id + # Get tenant/account ID + tenant_resp = run_carbide("tenant", "get") + result["account_id"] = tenant_resp.get("id", tenant_resp.get("tenant_id", "")) + result["success"] = True + else: + result["error"] = f"SSH key {key_id} not found in key listing" + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/control-plane/verify_key_rejected.py b/isvctl/configs/stubs/carbide/control-plane/verify_key_rejected.py new file mode 100644 index 00000000..0a02e995 --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/verify_key_rejected.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Verify the deleted SSH key no longer appears in Carbide. + +Carbide does not have a "reject" concept; instead we verify +that the previously deleted SSH key is absent from the key listing. + +Usage: + python verify_key_rejected.py --access-key-id + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "rejected": true, + "error_code": "KeyNotFound" +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Verify Carbide SSH key is gone") + parser.add_argument("--access-key-id", default="") + parser.add_argument("--secret-access-key", default="") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--wait", type=int, default=0, help="Seconds to wait (unused)") + parser.add_argument("--retries", type=int, default=3, help="Number of retry attempts") + args = parser.parse_args() + + state = load_state() + key_id = args.access_key_id or state.get("ssh_key_id", "") + + result: dict[str, Any] = { + "success": False, + "platform": "control_plane", + "rejected": False, + } + + try: + keys_resp = run_carbide("ssh-key", "list") + keys = keys_resp if isinstance(keys_resp, list) else keys_resp.get("items", []) + + found = any( + k.get("id", k.get("ssh_key_id", "")) == key_id + for k in keys + ) + + if not found: + result["rejected"] = True + result["error_code"] = "KeyNotFound" + result["success"] = True + else: + result["error"] = f"SSH key {key_id} still present after deletion" + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + # Always exit 0 - let validation check the 'rejected' field + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/iam/create_user.py b/isvctl/configs/stubs/carbide/iam/create_user.py new file mode 100644 index 00000000..0fddb9c0 --- /dev/null +++ b/isvctl/configs/stubs/carbide/iam/create_user.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Validate Carbide API token and extract scopes. + +Maps the IAM template's "create_user" step. Instead of creating a user +(Carbide users are managed via NGC/Keycloak), this validates the current +API token, extracts granted scopes, and creates a temporary SSH key to +prove write access works. + +Usage: + python create_user.py + +Output JSON: +{ + "success": true, + "platform": "iam", + "username": "", + "user_id": "", + "access_key_id": "", + "scopes": {"vpc": ["read", "write"], ...}, + "scope_count": 32 +} +""" + +import json +import os +import sys +import time +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import get_scopes, load_state, run_carbide, save_state, timed_call + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "iam", + } + + try: + # Validate token by calling tenant get + tenant_data, latency = timed_call("tenant", "get") + tenant_name = tenant_data.get("name", "") + tenant_id = tenant_data.get("id", "") + + result["username"] = tenant_name + result["user_id"] = tenant_id + result["auth_latency_ms"] = round(latency * 1000) + + # Extract scopes + scopes = get_scopes() + result["scopes"] = {k: sorted(v) for k, v in scopes.items()} + result["scope_count"] = sum(len(v) for v in scopes.values()) + + # Prove write access: create a temporary SSH key group + key + suffix = int(time.time()) + group_name = f"ncp-iam-test-{suffix}" + + state = load_state() + + group_resp = run_carbide("ssh-key-group", "create", "--name", group_name) + group_id = group_resp.get("id", "") + state["iam_ssh_key_group_id"] = group_id + state["iam_ssh_key_group_created"] = True + + key_resp = run_carbide( + "ssh-key", "create", + "--name", f"ncp-iam-key-{suffix}", + "--ssh-key-group-id", group_id, + ) + key_id = key_resp.get("id", "") + state["iam_ssh_key_id"] = key_id + + result["access_key_id"] = key_id + result["success"] = True + + save_state(state) + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/iam/delete_user.py b/isvctl/configs/stubs/carbide/iam/delete_user.py new file mode 100644 index 00000000..45b5996a --- /dev/null +++ b/isvctl/configs/stubs/carbide/iam/delete_user.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Clean up temporary SSH key created during IAM validation. + +Maps the IAM template's "teardown" step. Deletes the temporary SSH key +and key group created by create_user.py. + +Usage: + python delete_user.py + +Output JSON: +{ + "success": true, + "platform": "iam", + "resources_deleted": ["ssh-key/", "ssh-key-group/"] +} +""" + +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide, save_state + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "iam", + "resources_deleted": [], + } + + state = load_state() + + # Delete SSH key + key_id = state.get("iam_ssh_key_id", "") + if key_id: + try: + run_carbide("ssh-key", "delete", "--id", key_id) + result["resources_deleted"].append(f"ssh-key/{key_id}") + except RuntimeError as e: + if "not found" not in str(e).lower(): + result["error"] = str(e) + print(json.dumps(result, indent=2)) + return 1 + + # Delete SSH key group + group_id = state.get("iam_ssh_key_group_id", "") + if group_id and state.get("iam_ssh_key_group_created", True): + try: + run_carbide("ssh-key-group", "delete", "--id", group_id) + result["resources_deleted"].append(f"ssh-key-group/{group_id}") + except RuntimeError as e: + if "not found" not in str(e).lower(): + result["error"] = str(e) + print(json.dumps(result, indent=2)) + return 1 + + # Clean up state + for key in ("iam_ssh_key_id", "iam_ssh_key_group_id", "iam_ssh_key_group_created"): + state.pop(key, None) + save_state(state) + + result["success"] = True + result["message"] = "IAM test resources cleaned up" + + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/iam/test_credentials.py b/isvctl/configs/stubs/carbide/iam/test_credentials.py new file mode 100644 index 00000000..fc9834cf --- /dev/null +++ b/isvctl/configs/stubs/carbide/iam/test_credentials.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Verify Carbide API scopes against template requirements. + +Maps the IAM template's "test_credentials" step. Checks that the +current token has the required scopes for each Carbide template +(control-plane, network, image-registry, bm). + +Usage: + python test_credentials.py + +Output JSON: +{ + "success": true, + "platform": "iam", + "account_id": "", + "tests": { + "identity": {"passed": true, "message": "..."}, + "scopes_control-plane": {"passed": true, "granted": [...], "missing": []}, + "scopes_network": {"passed": true, ...}, + ... + } +} +""" + +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import ( + TEMPLATE_SCOPES, + check_scopes, + effective_scopes_for_template, + run_carbide, + timed_call, +) + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "iam", + "tests": {}, + } + + try: + # Identity check + tenant_data, latency = timed_call("tenant", "get") + result["account_id"] = tenant_data.get("id", "") + result["tests"]["identity"] = { + "passed": True, + "message": f"Authenticated as {tenant_data.get('name', '')}", + "latency_ms": round(latency * 1000), + } + + # Read access check + try: + run_carbide("site", "list") + result["tests"]["access"] = { + "passed": True, + "message": "Read access verified (site list)", + } + except Exception as e: + result["tests"]["access"] = { + "passed": False, + "message": f"Read access failed: {e}", + } + + # Scope checks per template (accounts for pre-existing resources) + all_passed = True + for template_name in TEMPLATE_SCOPES: + effective = effective_scopes_for_template(template_name) + ok, granted, missing = check_scopes(effective) + result["tests"][f"scopes_{template_name}"] = { + "passed": ok, + "granted": granted, + "missing": missing, + "effective_requirements": {k: v for k, v in effective.items()}, + } + if not ok: + all_passed = False + + result["success"] = ( + result["tests"]["identity"]["passed"] + and result["tests"]["access"]["passed"] + ) + # Note: scope check failures are warnings, not hard failures. + # The token works, but some templates may not run. + if not all_passed: + result["scope_warnings"] = True + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/image-registry/crud_install_config.py b/isvctl/configs/stubs/carbide/image-registry/crud_install_config.py new file mode 100644 index 00000000..045942ba --- /dev/null +++ b/isvctl/configs/stubs/carbide/image-registry/crud_install_config.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""OS install config CRUD lifecycle via Carbide operating-system API. + +Exercises get, list, update, and delete on the OperatingSystem resource +to validate full CRUD capability. Maps to the image-registry template's +``crud_install_config`` step. + +Usage: + python crud_install_config.py --os-id --site-id + +Output JSON: +{ + "success": true, + "platform": "image_registry", + "config_id": "", + "config_name": "ncp-test-config", + "operations": {"create": true, "read": true, "update": true, "delete": true} +} +""" + +import argparse +import json +import os +import sys +import time +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Carbide OS install config CRUD lifecycle") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "")) + parser.add_argument("--os-id", default=os.environ.get("CARBIDE_OS_ID", "")) + parser.add_argument("--site-id", default=os.environ.get("CARBIDE_SITE_ID", "")) + args = parser.parse_args() + + if not args.os_id: + state = load_state() + args.os_id = state.get("os_id", "") + + if not args.os_id: + print(json.dumps({ + "success": False, + "platform": "image_registry", + "error": "os-id is required (--os-id or from previous step state)", + }, indent=2)) + return 1 + + config_name = f"ncp-test-config-{int(time.time())}" + + result: dict[str, Any] = { + "success": False, + "platform": "image_registry", + "config_id": args.os_id, + "config_name": config_name, + "operations": {"create": False, "read": False, "update": False, "delete": False}, + } + + try: + # CREATE: A new OS config for CRUD testing + create_resp = run_carbide( + "operating-system", "create", + "--name", config_name, + "--site-id", args.site_id, + "--type", "ipxe", + ) + crud_os_id = create_resp.get("id", create_resp.get("operating_system_id", "")) + result["config_id"] = crud_os_id + result["operations"]["create"] = True + + # READ: Get the OS config + run_carbide("operating-system", "get", "--id", crud_os_id) + result["operations"]["read"] = True + + # UPDATE: Update the OS config description + run_carbide( + "operating-system", "update", + "--id", crud_os_id, + "--name", f"{config_name}-updated", + ) + result["operations"]["update"] = True + + # DELETE: Remove the CRUD test config + run_carbide("operating-system", "delete", "--id", crud_os_id) + result["operations"]["delete"] = True + + result["success"] = all(result["operations"].values()) + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/image-registry/install_config_bm.py b/isvctl/configs/stubs/carbide/image-registry/install_config_bm.py new file mode 100644 index 00000000..251183bc --- /dev/null +++ b/isvctl/configs/stubs/carbide/image-registry/install_config_bm.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Provision a bare-metal instance using OS config, proving OS+config works. + +Uses ``carbidecli instance create`` with the OS config to provision +a BM instance. Maps to the image-registry template's ``install_config_bm`` +step. + +Usage: + python install_config_bm.py --os-id --site-id + +Output JSON: +{ + "success": true, + "platform": "image_registry", + "instance_id": "", + "config_id": "", + "instance_state": "running" +} +""" + +import argparse +import json +import os +import sys +import time +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide, save_state + + +def wait_for_instance(instance_id: str, timeout: int = 300) -> dict[str, Any]: + """Poll instance status until running or timeout.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + resp = run_carbide("instance", "get", "--id", instance_id) + state = resp.get("status", resp.get("state", "")) + if state.lower() in ("running", "active"): + return resp + print(f"Instance {instance_id} state: {state}, waiting...", file=sys.stderr) + time.sleep(15) + raise RuntimeError(f"Instance {instance_id} did not reach running state within {timeout}s") + + +def main() -> int: + parser = argparse.ArgumentParser(description="Provision BM instance with OS config") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "")) + parser.add_argument("--os-id", default=os.environ.get("CARBIDE_OS_ID", "")) + parser.add_argument("--site-id", default=os.environ.get("CARBIDE_SITE_ID", "")) + parser.add_argument("--name", default="ncp-bm-cfg-test") + args = parser.parse_args() + + if not args.os_id: + state = load_state() + args.os_id = state.get("os_id", "") + + if not args.os_id or not args.site_id: + print(json.dumps({ + "success": False, + "platform": "image_registry", + "error": "os-id and site-id are required", + }, indent=2)) + return 1 + + instance_name = f"{args.name}-{int(time.time())}" + + result: dict[str, Any] = { + "success": False, + "platform": "image_registry", + "config_id": args.os_id, + } + + try: + resp = run_carbide( + "instance", "create", + "--name", instance_name, + "--operating-system-id", args.os_id, + "--site-id", args.site_id, + ) + instance_id = resp.get("id", resp.get("instance_id", "")) + result["instance_id"] = instance_id + + # Wait for instance to be running + wait_for_instance(instance_id) + result["instance_state"] = "running" + result["success"] = True + + # Track for teardown + state = load_state() + state["bm_cfg_instance_id"] = instance_id + state.setdefault("instance_ids", []).append(instance_id) + save_state(state) + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/image-registry/install_image_bm.py b/isvctl/configs/stubs/carbide/image-registry/install_image_bm.py new file mode 100644 index 00000000..21e255c5 --- /dev/null +++ b/isvctl/configs/stubs/carbide/image-registry/install_image_bm.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Provision a bare-metal instance with a specific OS image. + +Uses ``carbidecli instance create`` with the OS ID to provision +a BM instance, proving the OS image works on bare metal. +Maps to the image-registry template's ``install_image_bm`` step. + +Usage: + python install_image_bm.py --os-id --site-id + +Output JSON: +{ + "success": true, + "platform": "image_registry", + "instance_id": "", + "image_id": "", + "instance_state": "running" +} +""" + +import argparse +import json +import os +import sys +import time +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide, save_state + + +def wait_for_instance(instance_id: str, timeout: int = 300) -> dict[str, Any]: + """Poll instance status until running or timeout.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + resp = run_carbide("instance", "get", "--id", instance_id) + state = resp.get("status", resp.get("state", "")) + if state.lower() in ("running", "active"): + return resp + print(f"Instance {instance_id} state: {state}, waiting...", file=sys.stderr) + time.sleep(15) + raise RuntimeError(f"Instance {instance_id} did not reach running state within {timeout}s") + + +def main() -> int: + parser = argparse.ArgumentParser(description="Provision BM instance with OS image") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "")) + parser.add_argument("--os-id", default=os.environ.get("CARBIDE_OS_ID", "")) + parser.add_argument("--site-id", default=os.environ.get("CARBIDE_SITE_ID", "")) + parser.add_argument("--name", default="ncp-bm-img-test") + args = parser.parse_args() + + if not args.os_id: + state = load_state() + args.os_id = state.get("os_id", "") + + if not args.os_id or not args.site_id: + print(json.dumps({ + "success": False, + "platform": "image_registry", + "error": "os-id and site-id are required", + }, indent=2)) + return 1 + + instance_name = f"{args.name}-{int(time.time())}" + + result: dict[str, Any] = { + "success": False, + "platform": "image_registry", + "image_id": args.os_id, + } + + try: + resp = run_carbide( + "instance", "create", + "--name", instance_name, + "--operating-system-id", args.os_id, + "--site-id", args.site_id, + ) + instance_id = resp.get("id", resp.get("instance_id", "")) + result["instance_id"] = instance_id + + # Wait for instance to be running + wait_for_instance(instance_id) + result["instance_state"] = "running" + result["success"] = True + + # Track for teardown + state = load_state() + state["bm_img_instance_id"] = instance_id + state.setdefault("instance_ids", []).append(instance_id) + save_state(state) + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/image-registry/launch_instance.py b/isvctl/configs/stubs/carbide/image-registry/launch_instance.py new file mode 100644 index 00000000..9a58b759 --- /dev/null +++ b/isvctl/configs/stubs/carbide/image-registry/launch_instance.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Launch a Carbide instance using a previously created OS resource. + +Uses ``carbidecli instance create`` with the OS ID from the upload step. +Maps to the image-registry template's ``launch_instance`` step. + +Usage: + python launch_instance.py --os-id --site-id + +Output JSON: +{ + "success": true, + "platform": "image_registry", + "instance_id": "", + "public_ip": "...", + "private_ip": "...", + "state": "running", + "key_path": "" +} +""" + +import argparse +import json +import os +import sys +import time +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide, save_state + + +def wait_for_instance(instance_id: str, timeout: int = 300) -> dict[str, Any]: + """Poll instance status until running or timeout.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + resp = run_carbide("instance", "get", "--id", instance_id) + state = resp.get("status", resp.get("state", "")) + if state.lower() in ("running", "active"): + return resp + print(f"Instance {instance_id} state: {state}, waiting...", file=sys.stderr) + time.sleep(15) + raise RuntimeError(f"Instance {instance_id} did not reach running state within {timeout}s") + + +def main() -> int: + parser = argparse.ArgumentParser(description="Launch Carbide instance from OS image") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "")) + parser.add_argument("--os-id", default=os.environ.get("CARBIDE_OS_ID", "")) + parser.add_argument("--site-id", default=os.environ.get("CARBIDE_SITE_ID", "")) + parser.add_argument("--name", default="ncp-img-test") + args = parser.parse_args() + + if not args.os_id: + # Fall back to state file + state = load_state() + args.os_id = state.get("os_id", "") + + if not args.os_id or not args.site_id: + print(json.dumps({ + "success": False, + "platform": "image_registry", + "error": "os-id and site-id are required", + }, indent=2)) + return 1 + + instance_name = f"{args.name}-{int(time.time())}" + + result: dict[str, Any] = { + "success": False, + "platform": "image_registry", + } + + try: + resp = run_carbide( + "instance", "create", + "--name", instance_name, + "--operating-system-id", args.os_id, + "--site-id", args.site_id, + ) + instance_id = resp.get("id", resp.get("instance_id", "")) + result["instance_id"] = instance_id + + # Wait for instance to be running + instance_data = wait_for_instance(instance_id) + + result["public_ip"] = instance_data.get("public_ip", instance_data.get("ip_address", "")) + result["private_ip"] = instance_data.get("private_ip", instance_data.get("internal_ip", "")) + result["state"] = "running" + result["key_path"] = "" + result["success"] = True + + # Persist for subsequent steps + state = load_state() + state["img_instance_id"] = instance_id + state.setdefault("instance_ids", []).append(instance_id) + save_state(state) + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/image-registry/teardown.py b/isvctl/configs/stubs/carbide/image-registry/teardown.py new file mode 100644 index 00000000..68320591 --- /dev/null +++ b/isvctl/configs/stubs/carbide/image-registry/teardown.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Delete all instances and OS resources created during image-registry testing. + +Reads the state file to find all instance IDs and the OS ID, then +deletes them via ``carbidecli instance delete`` and +``carbidecli operating-system delete``. + +Usage: + python teardown.py --site-id + +Output JSON: +{ + "success": true, + "platform": "image_registry", + "resources_deleted": [...] +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide, save_state + + +def main() -> int: + parser = argparse.ArgumentParser(description="Teardown Carbide image-registry resources") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "")) + parser.add_argument("--site-id", default=os.environ.get("CARBIDE_SITE_ID", "")) + args = parser.parse_args() + + state = load_state() + resources_deleted: list[str] = [] + errors: list[str] = [] + + result: dict[str, Any] = { + "success": False, + "platform": "image_registry", + "resources_deleted": resources_deleted, + } + + # Delete all tracked instances + instance_ids = state.get("instance_ids", []) + for instance_id in instance_ids: + try: + run_carbide("instance", "delete", "--id", instance_id) + resources_deleted.append(f"instance:{instance_id}") + except Exception as e: + errors.append(f"instance:{instance_id}: {e}") + + # Delete the OS resource + os_id = state.get("os_id", "") + if os_id: + try: + run_carbide("operating-system", "delete", "--id", os_id) + resources_deleted.append(f"operating-system:{os_id}") + except Exception as e: + errors.append(f"operating-system:{os_id}: {e}") + + if errors: + result["errors"] = errors + + # Clear state + save_state({}) + + result["success"] = len(errors) == 0 + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/image-registry/upload_image.py b/isvctl/configs/stubs/carbide/image-registry/upload_image.py new file mode 100644 index 00000000..587d854e --- /dev/null +++ b/isvctl/configs/stubs/carbide/image-registry/upload_image.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Create an OperatingSystem resource in Carbide (image upload equivalent). + +Uses ``carbidecli operating-system create`` to register an OS with +iPXE/kickstart boot type. Maps to the image-registry template's +``upload_image`` step. + +Usage: + python upload_image.py --site-id + +Output JSON: +{ + "success": true, + "platform": "image_registry", + "image_id": "", + "image_name": "ncp-validation-os", + "storage_bucket": "carbide", + "disk_ids": [""] +} +""" + +import argparse +import json +import os +import sys +import time +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide, save_state + + +def main() -> int: + parser = argparse.ArgumentParser(description="Create Carbide OS resource") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "")) + parser.add_argument("--site-id", default=os.environ.get("CARBIDE_SITE_ID", "")) + parser.add_argument("--os-name", default="ncp-validation-os") + parser.add_argument("--os-id", default=os.environ.get("CARBIDE_OS_ID", "")) + args = parser.parse_args() + + result: dict[str, Any] = { + "success": False, + "platform": "image_registry", + } + + try: + state = load_state() + + if args.os_id: + # Use pre-existing OperatingSystem + resp = run_carbide("operating-system", "get", args.os_id) + os_id = resp.get("id", args.os_id) + os_name = resp.get("name", args.os_id) + state["os_created"] = False + else: + # Create new OperatingSystem + if not args.site_id: + result["error"] = "site-id required when not using pre-existing OS" + print(json.dumps(result, indent=2)) + return 1 + os_name = f"{args.os_name}-{int(time.time())}" + resp = run_carbide( + "operating-system", "create", + "--name", os_name, + "--site-id", args.site_id, + "--type", "ipxe", + ) + os_id = resp.get("id", resp.get("operating_system_id", "")) + state["os_created"] = True + + result["image_id"] = os_id + result["image_name"] = os_name + result["storage_bucket"] = "carbide" + result["disk_ids"] = [os_id] + result["success"] = True + + state["os_id"] = os_id + state["os_name"] = os_name + state["site_id"] = args.site_id + save_state(state) + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/network/create_vpc.py b/isvctl/configs/stubs/carbide/network/create_vpc.py new file mode 100644 index 00000000..d499945e --- /dev/null +++ b/isvctl/configs/stubs/carbide/network/create_vpc.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Create or reuse a VPC with prefix and subnet in Carbide. + +Sets up the base network resources used by subsequent test steps. +Supports pre-existing resources via environment variables. + +Usage: + python create_vpc.py --site-id + CARBIDE_VPC_ID= CARBIDE_VPC_PREFIX_ID= CARBIDE_SUBNET_ID= python create_vpc.py + +Output JSON: +{ + "success": true, + "platform": "network", + "network_id": "", + "cidr": "10.100.0.0/24", + "subnets": [{"subnet_id": "...", "cidr": "..."}] +} +""" + +import argparse +import json +import os +import sys +import time +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide, save_state + + +def main() -> int: + parser = argparse.ArgumentParser(description="Create or reuse Carbide VPC + prefix + subnet") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--site-id", default=os.environ.get("CARBIDE_SITE_ID", "")) + parser.add_argument("--vpc-id", default=os.environ.get("CARBIDE_VPC_ID", "")) + parser.add_argument("--vpc-prefix-id", default=os.environ.get("CARBIDE_VPC_PREFIX_ID", "")) + parser.add_argument("--subnet-id", default=os.environ.get("CARBIDE_SUBNET_ID", "")) + args = parser.parse_args() + + cidr = "10.100.0.0/24" + subnet_cidr = "10.100.0.0/26" + + result: dict[str, Any] = { + "success": False, + "platform": "network", + } + + try: + state = load_state() + + # VPC: reuse or create + if args.vpc_id: + vpc_resp = run_carbide("vpc", "get", args.vpc_id) + vpc_id = vpc_resp.get("id", args.vpc_id) + vpc_name = vpc_resp.get("name", args.vpc_id) + state["network_vpc_created"] = False + else: + if not args.site_id: + result["error"] = "site-id required when not using pre-existing VPC" + print(json.dumps(result, indent=2)) + return 1 + vpc_name = f"ncp-net-{int(time.time())}" + vpc_resp = run_carbide( + "vpc", "create", + "--name", vpc_name, + "--description", "NCP network validation VPC", + "--site-id", args.site_id, + ) + vpc_id = vpc_resp.get("id", vpc_resp.get("vpc_id", "")) + state["network_vpc_created"] = True + + # VPC Prefix: reuse or create + if args.vpc_prefix_id: + prefix_id = args.vpc_prefix_id + state["network_prefix_created"] = False + else: + prefix_resp = run_carbide( + "vpc-prefix", "create", + "--vpc-id", vpc_id, + "--cidr", cidr, + ) + prefix_id = prefix_resp.get("id", prefix_resp.get("prefix_id", "")) + state["network_prefix_created"] = True + + # Subnet: reuse or create + if args.subnet_id: + subnet_id = args.subnet_id + state["network_subnet_created"] = False + else: + subnet_resp = run_carbide( + "subnet", "create", + "--vpc-id", vpc_id, + "--cidr", subnet_cidr, + "--name", f"{vpc_name}-subnet-0", + ) + subnet_id = subnet_resp.get("id", subnet_resp.get("subnet_id", "")) + state["network_subnet_created"] = True + + result["network_id"] = vpc_id + result["cidr"] = cidr + result["subnets"] = [{"subnet_id": subnet_id, "cidr": subnet_cidr}] + result["success"] = True + + state["network_vpc_id"] = vpc_id + state["network_vpc_name"] = vpc_name + state["network_prefix_id"] = prefix_id + state["network_subnet_ids"] = [subnet_id] + state["site_id"] = args.site_id + save_state(state) + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/network/isolation_test.py b/isvctl/configs/stubs/carbide/network/isolation_test.py new file mode 100644 index 00000000..a2c8c3f0 --- /dev/null +++ b/isvctl/configs/stubs/carbide/network/isolation_test.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Test VPC isolation in Carbide. + +Creates two separate VPCs and verifies they cannot see each other +(no cross-references in listings). + +Usage: + python isolation_test.py --site-id + +Output JSON: +{ + "success": true, + "platform": "network", + "tests": { + "create_vpc_a": {"passed": true}, + "create_vpc_b": {"passed": true}, + "no_peering": {"passed": true} + } +} +""" + +import argparse +import json +import os +import sys +import time +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Carbide VPC isolation test") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--site-id", default=os.environ.get("CARBIDE_SITE_ID", "")) + args = parser.parse_args() + + if not args.site_id: + print(json.dumps({ + "success": False, + "platform": "network", + "error": "site-id is required (--site-id or CARBIDE_SITE_ID env var)", + }, indent=2)) + return 1 + + ts = int(time.time()) + vpc_a_id = "" + vpc_b_id = "" + + result: dict[str, Any] = { + "success": False, + "platform": "network", + "tests": {}, + } + + try: + # Create VPC A + try: + resp_a = run_carbide( + "vpc", "create", + "--name", f"ncp-iso-a-{ts}", + "--description", "NCP isolation test VPC A", + "--site-id", args.site_id, + ) + vpc_a_id = resp_a.get("id", resp_a.get("vpc_id", "")) + result["tests"]["create_vpc_a"] = {"passed": bool(vpc_a_id)} + except Exception as e: + result["tests"]["create_vpc_a"] = {"passed": False, "error": str(e)} + + # Create VPC B + try: + resp_b = run_carbide( + "vpc", "create", + "--name", f"ncp-iso-b-{ts}", + "--description", "NCP isolation test VPC B", + "--site-id", args.site_id, + ) + vpc_b_id = resp_b.get("id", resp_b.get("vpc_id", "")) + result["tests"]["create_vpc_b"] = {"passed": bool(vpc_b_id)} + except Exception as e: + result["tests"]["create_vpc_b"] = {"passed": False, "error": str(e)} + + # Verify isolation: subnets in VPC A should not appear in VPC B listing + if vpc_a_id and vpc_b_id: + try: + subnets_a = run_carbide("subnet", "list", "--vpc-id", vpc_a_id) + subnets_b = run_carbide("subnet", "list", "--vpc-id", vpc_b_id) + + list_a = subnets_a if isinstance(subnets_a, list) else subnets_a.get("subnets", []) + list_b = subnets_b if isinstance(subnets_b, list) else subnets_b.get("subnets", []) + + ids_a = {s.get("id", s.get("subnet_id", "")) for s in list_a} + ids_b = {s.get("id", s.get("subnet_id", "")) for s in list_b} + no_overlap = ids_a.isdisjoint(ids_b) + result["tests"]["no_peering"] = {"passed": no_overlap} + except Exception as e: + result["tests"]["no_peering"] = {"passed": False, "error": str(e)} + else: + result["tests"]["no_peering"] = {"passed": False, "error": "could not create both VPCs"} + + all_passed = all(t.get("passed", False) for t in result["tests"].values()) + result["success"] = all_passed + + finally: + # Clean up both VPCs + for vpc_id in (vpc_a_id, vpc_b_id): + if vpc_id: + try: + run_carbide("vpc", "delete", "--id", vpc_id) + except Exception: + pass + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/network/security_test.py b/isvctl/configs/stubs/carbide/network/security_test.py new file mode 100644 index 00000000..896262cb --- /dev/null +++ b/isvctl/configs/stubs/carbide/network/security_test.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Test NSG security rules in Carbide. + +Creates a VPC with a Network Security Group and rules, then verifies +the rules exist and are configured for default-deny inbound. + +Usage: + python security_test.py --site-id + +Output JSON: +{ + "success": true, + "platform": "network", + "tests": { + "create_vpc": {"passed": true}, + "sg_default_deny_inbound": {"passed": true} + } +} +""" + +import argparse +import json +import os +import sys +import time +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Carbide NSG security test") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--site-id", default=os.environ.get("CARBIDE_SITE_ID", "")) + args = parser.parse_args() + + if not args.site_id: + print(json.dumps({ + "success": False, + "platform": "network", + "error": "site-id is required (--site-id or CARBIDE_SITE_ID env var)", + }, indent=2)) + return 1 + + ts = int(time.time()) + vpc_id = "" + nsg_id = "" + + result: dict[str, Any] = { + "success": False, + "platform": "network", + "tests": {}, + } + + try: + # Create VPC + try: + vpc_resp = run_carbide( + "vpc", "create", + "--name", f"ncp-sec-{ts}", + "--description", "NCP security test VPC", + "--site-id", args.site_id, + ) + vpc_id = vpc_resp.get("id", vpc_resp.get("vpc_id", "")) + result["tests"]["create_vpc"] = {"passed": bool(vpc_id)} + except Exception as e: + result["tests"]["create_vpc"] = {"passed": False, "error": str(e)} + + # Create NSG and verify default-deny + if vpc_id: + try: + nsg_resp = run_carbide( + "network-security-group", "create", + "--vpc-id", vpc_id, + "--name", f"ncp-nsg-{ts}", + ) + nsg_id = nsg_resp.get("id", nsg_resp.get("nsg_id", "")) + + # Get NSG details and check rules + nsg_detail = run_carbide( + "network-security-group", "get", + "--id", nsg_id, + ) + + # Verify NSG exists and has rules (default deny inbound) + rules = nsg_detail.get("rules", nsg_detail.get("security_rules", [])) + has_deny = any( + r.get("action", "").lower() == "deny" + or r.get("direction", "").lower() == "inbound" + for r in rules + ) if rules else True # No rules = implicit deny + + result["tests"]["sg_default_deny_inbound"] = {"passed": bool(nsg_id) and has_deny} + except Exception as e: + result["tests"]["sg_default_deny_inbound"] = {"passed": False, "error": str(e)} + else: + result["tests"]["sg_default_deny_inbound"] = { + "passed": False, "error": "no vpc_id from create", + } + + all_passed = all(t.get("passed", False) for t in result["tests"].values()) + result["success"] = all_passed + + finally: + # Clean up + if nsg_id: + try: + run_carbide("network-security-group", "delete", "--id", nsg_id) + except Exception: + pass + if vpc_id: + try: + run_carbide("vpc", "delete", "--id", vpc_id) + except Exception: + pass + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/network/subnet_test.py b/isvctl/configs/stubs/carbide/network/subnet_test.py new file mode 100644 index 00000000..90b3000c --- /dev/null +++ b/isvctl/configs/stubs/carbide/network/subnet_test.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Test subnet creation in the shared Carbide VPC. + +Creates multiple subnets in the VPC from the setup step and verifies +they are available. + +Usage: + python subnet_test.py --site-id + +Output JSON: +{ + "success": true, + "platform": "network", + "subnets": [...], + "tests": { + "create_subnets": {"passed": true}, + "subnets_available": {"passed": true} + } +} +""" + +import argparse +import json +import os +import sys +import time +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide, save_state + + +# Subnet CIDRs within the 10.100.0.0/24 prefix +SUBNET_CIDRS = [ + "10.100.0.64/26", + "10.100.0.128/26", + "10.100.0.192/26", +] + + +def main() -> int: + parser = argparse.ArgumentParser(description="Carbide subnet test") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--site-id", default=os.environ.get("CARBIDE_SITE_ID", "")) + args = parser.parse_args() + + state = load_state() + vpc_id = state.get("network_vpc_id", "") + + result: dict[str, Any] = { + "success": False, + "platform": "network", + "subnets": [], + "tests": {}, + } + + if not vpc_id: + result["error"] = "no network_vpc_id in state; run create_vpc first" + print(json.dumps(result, indent=2)) + return 1 + + created_ids: list[str] = [] + + try: + # CREATE subnets + ts = int(time.time()) + for i, cidr in enumerate(SUBNET_CIDRS): + resp = run_carbide( + "subnet", "create", + "--vpc-id", vpc_id, + "--cidr", cidr, + "--name", f"ncp-subnet-{ts}-{i}", + ) + sid = resp.get("id", resp.get("subnet_id", "")) + created_ids.append(sid) + result["subnets"].append({"subnet_id": sid, "cidr": cidr}) + + result["tests"]["create_subnets"] = {"passed": len(created_ids) == len(SUBNET_CIDRS)} + + # VERIFY subnets via list + list_resp = run_carbide("subnet", "list", "--vpc-id", vpc_id) + listed = list_resp if isinstance(list_resp, list) else list_resp.get("subnets", []) + listed_ids = {s.get("id", s.get("subnet_id", "")) for s in listed} + all_found = all(sid in listed_ids for sid in created_ids) + result["tests"]["subnets_available"] = {"passed": all_found} + + all_passed = all(t.get("passed", False) for t in result["tests"].values()) + result["success"] = all_passed + + # Persist created subnet IDs for teardown + existing = state.get("network_subnet_ids", []) + state["network_subnet_ids"] = existing + created_ids + save_state(state) + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/network/teardown.py b/isvctl/configs/stubs/carbide/network/teardown.py new file mode 100644 index 00000000..55d88555 --- /dev/null +++ b/isvctl/configs/stubs/carbide/network/teardown.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Tear down all Carbide network resources created during validation. + +Deletes subnets, VPC prefixes, NSGs, and VPCs tracked in state. + +Usage: + python teardown.py --site-id + +Output JSON: +{ + "success": true, + "platform": "network", + "resources_deleted": ["subnet/", "vpc-prefix/", "vpc/"] +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide, save_state + + +def _safe_delete(resource_type: str, resource_id: str) -> bool: + """Attempt to delete a resource, returning True on success or already-deleted.""" + if not resource_id: + return False + try: + run_carbide(resource_type, "delete", "--id", resource_id) + return True + except RuntimeError as e: + if "not found" in str(e).lower() or "404" in str(e): + return True + return False + + +def main() -> int: + parser = argparse.ArgumentParser(description="Carbide network teardown") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--site-id", default=os.environ.get("CARBIDE_SITE_ID", "")) + parser.add_argument("--skip-destroy", action="store_true") + args = parser.parse_args() + + result: dict[str, Any] = { + "success": False, + "platform": "network", + "resources_deleted": [], + } + + if args.skip_destroy: + result["success"] = True + result["skipped"] = True + print(json.dumps(result, indent=2)) + return 0 + + state = load_state() + + # Delete subnets (only those we created) + if state.get("network_subnet_created", True): + for subnet_id in state.get("network_subnet_ids", []): + if _safe_delete("subnet", subnet_id): + result["resources_deleted"].append(f"subnet/{subnet_id}") + + # Delete NSGs (always created by tests) + for nsg_id in state.get("network_nsg_ids", []): + if _safe_delete("network-security-group", nsg_id): + result["resources_deleted"].append(f"nsg/{nsg_id}") + + # Delete VPC prefix (only if we created it) + prefix_id = state.get("network_prefix_id", "") + if state.get("network_prefix_created", True) and prefix_id: + if _safe_delete("vpc-prefix", prefix_id): + result["resources_deleted"].append(f"vpc-prefix/{prefix_id}") + + # Delete VPC (only if we created it) + vpc_id = state.get("network_vpc_id", "") + if state.get("network_vpc_created", True) and vpc_id: + if _safe_delete("vpc", vpc_id): + result["resources_deleted"].append(f"vpc/{vpc_id}") + + # Clean up state keys + for key in ("network_vpc_id", "network_vpc_name", "network_prefix_id", + "network_subnet_ids", "network_nsg_ids"): + state.pop(key, None) + save_state(state) + + result["success"] = True + + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/network/test_connectivity.py b/isvctl/configs/stubs/carbide/network/test_connectivity.py new file mode 100644 index 00000000..729f9166 --- /dev/null +++ b/isvctl/configs/stubs/carbide/network/test_connectivity.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Verify subnet exists and is accessible in Carbide. + +Checks the shared VPC's subnet is present and in an active state, +confirming an instance would receive an IP from the subnet. + +Usage: + python test_connectivity.py --site-id + +Output JSON: +{ + "success": true, + "platform": "network", + "tests": { + "network_assigned": {"passed": true} + } +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Carbide connectivity test") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--site-id", default=os.environ.get("CARBIDE_SITE_ID", "")) + args = parser.parse_args() + + state = load_state() + vpc_id = state.get("network_vpc_id", "") + subnet_ids = state.get("network_subnet_ids", []) + + result: dict[str, Any] = { + "success": False, + "platform": "network", + "tests": {}, + } + + if not vpc_id: + result["error"] = "no network_vpc_id in state; run create_vpc first" + print(json.dumps(result, indent=2)) + return 1 + + try: + # Verify at least one subnet exists and is accessible + list_resp = run_carbide("subnet", "list", "--vpc-id", vpc_id) + listed = list_resp if isinstance(list_resp, list) else list_resp.get("subnets", []) + listed_ids = {s.get("id", s.get("subnet_id", "")) for s in listed} + + has_subnet = bool(listed_ids) + known_found = any(sid in listed_ids for sid in subnet_ids) if subnet_ids else has_subnet + + result["tests"]["network_assigned"] = {"passed": has_subnet and known_found} + result["success"] = has_subnet and known_found + + except Exception as e: + result["tests"]["network_assigned"] = {"passed": False, "error": str(e)} + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/network/traffic_test.py b/isvctl/configs/stubs/carbide/network/traffic_test.py new file mode 100644 index 00000000..26e28040 --- /dev/null +++ b/isvctl/configs/stubs/carbide/network/traffic_test.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Simplified traffic validation for Carbide. + +Verifies VPC and subnet routing configuration is correct by confirming +the VPC exists and subnets are properly associated. + +Usage: + python traffic_test.py --site-id + +Output JSON: +{ + "success": true, + "platform": "network", + "tests": { + "network_setup": {"passed": true} + } +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Carbide traffic validation") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--site-id", default=os.environ.get("CARBIDE_SITE_ID", "")) + args = parser.parse_args() + + state = load_state() + vpc_id = state.get("network_vpc_id", "") + + result: dict[str, Any] = { + "success": False, + "platform": "network", + "tests": {}, + } + + if not vpc_id: + result["error"] = "no network_vpc_id in state; run create_vpc first" + print(json.dumps(result, indent=2)) + return 1 + + try: + # Verify VPC exists + vpc_resp = run_carbide("vpc", "get", "--id", vpc_id) + got_id = vpc_resp.get("id", vpc_resp.get("vpc_id", "")) + + # Verify subnets are associated + list_resp = run_carbide("subnet", "list", "--vpc-id", vpc_id) + listed = list_resp if isinstance(list_resp, list) else list_resp.get("subnets", []) + + vpc_ok = got_id == vpc_id + subnets_ok = len(listed) > 0 + + result["tests"]["network_setup"] = {"passed": vpc_ok and subnets_ok} + result["success"] = vpc_ok and subnets_ok + + except Exception as e: + result["tests"]["network_setup"] = {"passed": False, "error": str(e)} + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/network/vpc_crud_test.py b/isvctl/configs/stubs/carbide/network/vpc_crud_test.py new file mode 100644 index 00000000..0155736a --- /dev/null +++ b/isvctl/configs/stubs/carbide/network/vpc_crud_test.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Test VPC CRUD lifecycle in Carbide. + +Creates a temporary VPC, reads it back, updates its tag, then deletes it. + +Usage: + python vpc_crud_test.py --site-id + +Output JSON: +{ + "success": true, + "platform": "network", + "tests": { + "create_vpc": {"passed": true}, + "read_vpc": {"passed": true}, + "delete_vpc": {"passed": true} + } +} +""" + +import argparse +import json +import os +import sys +import time +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Carbide VPC CRUD test") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--site-id", default=os.environ.get("CARBIDE_SITE_ID", "")) + args = parser.parse_args() + + if not args.site_id: + print(json.dumps({ + "success": False, + "platform": "network", + "error": "site-id is required (--site-id or CARBIDE_SITE_ID env var)", + }, indent=2)) + return 1 + + vpc_name = f"ncp-crud-{int(time.time())}" + vpc_id = "" + + result: dict[str, Any] = { + "success": False, + "platform": "network", + "tests": {}, + } + + try: + # CREATE + try: + resp = run_carbide( + "vpc", "create", + "--name", vpc_name, + "--description", "NCP CRUD test VPC", + "--site-id", args.site_id, + ) + vpc_id = resp.get("id", resp.get("vpc_id", "")) + result["tests"]["create_vpc"] = {"passed": bool(vpc_id)} + except Exception as e: + result["tests"]["create_vpc"] = {"passed": False, "error": str(e)} + + # READ + if vpc_id: + try: + get_resp = run_carbide("vpc", "get", "--id", vpc_id) + got_id = get_resp.get("id", get_resp.get("vpc_id", "")) + result["tests"]["read_vpc"] = {"passed": got_id == vpc_id} + except Exception as e: + result["tests"]["read_vpc"] = {"passed": False, "error": str(e)} + else: + result["tests"]["read_vpc"] = {"passed": False, "error": "no vpc_id from create"} + + # DELETE + if vpc_id: + try: + run_carbide("vpc", "delete", "--id", vpc_id) + result["tests"]["delete_vpc"] = {"passed": True} + except Exception as e: + result["tests"]["delete_vpc"] = {"passed": False, "error": str(e)} + else: + result["tests"]["delete_vpc"] = {"passed": False, "error": "no vpc_id to delete"} + + all_passed = all(t.get("passed", False) for t in result["tests"].values()) + result["success"] = all_passed + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/common/k8s-inventory.sh b/isvctl/configs/stubs/common/k8s-inventory.sh new file mode 100755 index 00000000..4b4393f3 --- /dev/null +++ b/isvctl/configs/stubs/common/k8s-inventory.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# Shared Kubernetes GPU Cluster Inventory +# +# Collects cluster inventory using kubectl (works on any K8s distribution). +# Sourced by provider-specific setup scripts. Sets shell variables that +# the caller uses to build its JSON output. +# +# Prerequisites: kubectl configured, jq available +# +# Variables set by this script: +# NODE_COUNT, NODES (JSON array) +# GPU_NODE_COUNT, GPU_PER_NODE, TOTAL_GPUS +# GPU_OPERATOR_NS +# DRIVER_VERSION +# RUNTIME_CLASS +# GPU_PRODUCT +# API_SERVER, KUBECONFIG_PATH + +# ----------------------------------------------------------------------------- +# Dependency Checks +# ----------------------------------------------------------------------------- + +for cmd in kubectl jq; do + if ! command -v "$cmd" &> /dev/null; then + echo "Error: $cmd not found" >&2 + exit 1 + fi +done + +if ! kubectl cluster-info &> /dev/null; then + echo "Error: Cannot connect to cluster. Check KUBECONFIG." >&2 + exit 1 +fi + +echo "Connected to cluster." >&2 + +# ----------------------------------------------------------------------------- +# Cluster Info +# ----------------------------------------------------------------------------- + +API_SERVER=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null || echo "") +KUBECONFIG_PATH="${KUBECONFIG:-$HOME/.kube/config}" + +# ----------------------------------------------------------------------------- +# Node Inventory +# ----------------------------------------------------------------------------- + +NODE_COUNT=$(kubectl get nodes --no-headers 2>/dev/null | wc -l | tr -d ' ') +NODES=$(kubectl get nodes -o json 2>/dev/null | jq '[.items[] | { + name: .metadata.name, + ready: (.status.conditions[] | select(.type=="Ready") | .status == "True"), + gpus: (if .status.capacity["nvidia.com/gpu"] then (.status.capacity["nvidia.com/gpu"] | tonumber) else 0 end) +}]' 2>/dev/null || echo '[]') + +echo "Nodes: ${NODE_COUNT}" >&2 + +# ----------------------------------------------------------------------------- +# GPU Inventory +# ----------------------------------------------------------------------------- + +GPU_NODE_COUNT=$(kubectl get nodes -l nvidia.com/gpu.present=true --no-headers 2>/dev/null | wc -l | tr -d ' ') +GPU_PER_NODE=$(kubectl get nodes -l nvidia.com/gpu.present=true -o jsonpath='{.items[0].status.capacity.nvidia\.com/gpu}' 2>/dev/null || echo "0") +GPU_PER_NODE=${GPU_PER_NODE:-0} +[ -z "$GPU_PER_NODE" ] || [ "$GPU_PER_NODE" = "null" ] && GPU_PER_NODE=0 +TOTAL_GPUS=$((GPU_NODE_COUNT * GPU_PER_NODE)) + +echo "GPU nodes: ${GPU_NODE_COUNT}, GPUs per node: ${GPU_PER_NODE}, Total: ${TOTAL_GPUS}" >&2 + +# ----------------------------------------------------------------------------- +# GPU Operator Namespace (check common conventions) +# ----------------------------------------------------------------------------- + +GPU_OPERATOR_NS="${GPU_OPERATOR_NS:-}" +if [ -z "$GPU_OPERATOR_NS" ]; then + for ns in nvidia-gpu-operator gpu-operator gpu-operator-resources openshift-operators; do + if kubectl get namespace "$ns" &> /dev/null; then + POD_COUNT=$(kubectl get pods -n "$ns" -l app=gpu-operator --no-headers 2>/dev/null | wc -l | tr -d ' ') + if [ "${POD_COUNT}" -gt 0 ]; then + GPU_OPERATOR_NS="$ns" + break + fi + fi + done + GPU_OPERATOR_NS="${GPU_OPERATOR_NS:-gpu-operator}" +fi + +echo "GPU Operator namespace: ${GPU_OPERATOR_NS}" >&2 + +# ----------------------------------------------------------------------------- +# Driver Version +# ----------------------------------------------------------------------------- + +DRIVER_MAJOR=$(kubectl get nodes -l nvidia.com/gpu.present=true -o jsonpath='{.items[0].metadata.labels.nvidia\.com/cuda\.driver\.major}' 2>/dev/null || echo "") +DRIVER_MINOR=$(kubectl get nodes -l nvidia.com/gpu.present=true -o jsonpath='{.items[0].metadata.labels.nvidia\.com/cuda\.driver\.minor}' 2>/dev/null || echo "") +DRIVER_REV=$(kubectl get nodes -l nvidia.com/gpu.present=true -o jsonpath='{.items[0].metadata.labels.nvidia\.com/cuda\.driver\.rev}' 2>/dev/null || echo "") + +if [ -n "$DRIVER_MAJOR" ] && [ -n "$DRIVER_MINOR" ] && [ -n "$DRIVER_REV" ]; then + DRIVER_VERSION="${DRIVER_MAJOR}.${DRIVER_MINOR}.${DRIVER_REV}" +elif [ -n "$DRIVER_MAJOR" ] && [ -n "$DRIVER_MINOR" ]; then + DRIVER_VERSION="${DRIVER_MAJOR}.${DRIVER_MINOR}" +else + DRIVER_VERSION="" +fi + +# ----------------------------------------------------------------------------- +# RuntimeClass +# ----------------------------------------------------------------------------- + +RUNTIME_CLASS="${RUNTIME_CLASS:-}" +if [ -z "$RUNTIME_CLASS" ]; then + kubectl get runtimeclass nvidia &> /dev/null 2>&1 && RUNTIME_CLASS="nvidia" +fi + +# ----------------------------------------------------------------------------- +# GPU Product Info +# ----------------------------------------------------------------------------- + +GPU_PRODUCT=$(kubectl get nodes -l nvidia.com/gpu.present=true -o jsonpath='{.items[0].metadata.labels.nvidia\.com/gpu\.product}' 2>/dev/null || echo "") + +echo "Driver: ${DRIVER_VERSION}, GPU: ${GPU_PRODUCT}, RuntimeClass: ${RUNTIME_CLASS:-none}" >&2 diff --git a/isvctl/configs/stubs/openshift/arm/arch_check.py b/isvctl/configs/stubs/openshift/arm/arch_check.py new file mode 100644 index 00000000..bc28fd76 --- /dev/null +++ b/isvctl/configs/stubs/openshift/arm/arch_check.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Verify nodes are running aarch64 architecture.""" + +import json +import subprocess +import sys +from typing import Any + + +def main() -> int: + result: dict[str, Any] = {"success": False, "platform": "openshift", "architecture": ""} + + r = subprocess.run( + ["kubectl", "get", "nodes", "-o", + "jsonpath={.items[*].status.nodeInfo.architecture}"], + capture_output=True, text=True, timeout=30, + ) + if r.returncode != 0: + result["error"] = f"Failed to get node architecture: {r.stderr}" + print(json.dumps(result, indent=2)) + return 1 + + archs = set(r.stdout.strip().split()) + result["architectures_found"] = sorted(archs) + + if "arm64" in archs or "aarch64" in archs: + result["architecture"] = "arm64" + result["success"] = True + elif "amd64" in archs: + result["architecture"] = "amd64" + result["success"] = True + result["skipped"] = True + result["info"] = "x86_64 cluster — ARM checks not applicable" + else: + result["architecture"] = ",".join(archs) + result["success"] = True + + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/arm/gicv4_check.py b/isvctl/configs/stubs/openshift/arm/gicv4_check.py new file mode 100644 index 00000000..6b2e342c --- /dev/null +++ b/isvctl/configs/stubs/openshift/arm/gicv4_check.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Check GICv4.1 (Generic Interrupt Controller) on ARM nodes.""" + +import json +import subprocess +import sys +from typing import Any + + +def is_arm() -> bool: + r = subprocess.run( + ["kubectl", "get", "nodes", "-o", "jsonpath={.items[0].status.nodeInfo.architecture}"], + capture_output=True, text=True, timeout=30, + ) + return r.returncode == 0 and r.stdout.strip() in ("arm64", "aarch64") + + +def main() -> int: + result: dict[str, Any] = {"success": False, "platform": "openshift", "gic_detected": False} + + if not is_arm(): + result["success"] = True + result["skipped"] = True + result["info"] = "Not an ARM cluster — GIC check skipped" + print(json.dumps(result, indent=2)) + return 0 + + node = subprocess.run( + ["kubectl", "get", "nodes", "-o", "jsonpath={.items[0].metadata.name}"], + capture_output=True, text=True, timeout=30, + ).stdout.strip() + + # Check for GICv4 in dmesg + r = subprocess.run( + ["oc", "debug", f"node/{node}", "--", "chroot", "/host", + "bash", "-c", "dmesg | grep -i 'gic\\|its' | head -10"], + capture_output=True, text=True, timeout=60, + ) + gic_output = r.stdout.strip() if r.returncode == 0 else "" + + if "gicv4" in gic_output.lower() or "GICv4" in gic_output: + result["gic_detected"] = True + result["gic_version"] = "v4.1" if "4.1" in gic_output else "v4" + elif "gicv3" in gic_output.lower(): + result["gic_detected"] = True + result["gic_version"] = "v3" + + result["dmesg_output"] = gic_output[:500] + + # Check /proc/interrupts for ITS + r = subprocess.run( + ["oc", "debug", f"node/{node}", "--", "chroot", "/host", + "bash", "-c", "ls /sys/firmware/devicetree/base/interrupt-controller/compatible 2>/dev/null && cat /sys/firmware/devicetree/base/interrupt-controller/compatible 2>/dev/null || echo none"], + capture_output=True, text=True, timeout=60, + ) + result["interrupt_controller"] = r.stdout.strip()[:200] if r.returncode == 0 else "" + + result["success"] = result["gic_detected"] + if not result["success"]: + result["success"] = True + result["warning"] = "GIC version not detected from dmesg — may need deeper inspection" + + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/arm/hugepage_check.py b/isvctl/configs/stubs/openshift/arm/hugepage_check.py new file mode 100644 index 00000000..b8a44dbd --- /dev/null +++ b/isvctl/configs/stubs/openshift/arm/hugepage_check.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Check hugepage configuration on nodes.""" + +import json +import subprocess +import sys +from typing import Any + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "hugepages_available": False, + } + + # Check hugepage capacity from node status + r = subprocess.run( + ["kubectl", "get", "nodes", "-o", + "jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.status.capacity.hugepages-1Gi}{'\\t'}{.status.capacity.hugepages-2Mi}{'\\n'}{end}"], + capture_output=True, text=True, timeout=30, + ) + if r.returncode != 0: + result["error"] = f"Failed to query nodes: {r.stderr}" + print(json.dumps(result, indent=2)) + return 1 + + node_hugepages = {} + for line in r.stdout.strip().split("\n"): + if not line.strip(): + continue + parts = line.split("\t") + if len(parts) >= 3: + name = parts[0] + hp_1g = parts[1] if parts[1] else "0" + hp_2m = parts[2] if parts[2] else "0" + node_hugepages[name] = {"1Gi": hp_1g, "2Mi": hp_2m} + + result["node_hugepages"] = node_hugepages + result["nodes_checked"] = len(node_hugepages) + + # Check if any node has hugepages + has_1g = any(v.get("1Gi", "0") not in ("0", "") for v in node_hugepages.values()) + has_2m = any(v.get("2Mi", "0") not in ("0", "") for v in node_hugepages.values()) + + result["hugepages_1gi_available"] = has_1g + result["hugepages_2mi_available"] = has_2m + result["hugepages_available"] = has_1g or has_2m + result["success"] = True # Report status, don't fail if no hugepages + + if not result["hugepages_available"]: + result["info"] = "No hugepages configured — may be needed for VFIO passthrough" + + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/arm/smmu_check.py b/isvctl/configs/stubs/openshift/arm/smmu_check.py new file mode 100644 index 00000000..8fd04768 --- /dev/null +++ b/isvctl/configs/stubs/openshift/arm/smmu_check.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Check SMMUv3 (System MMU) is available on ARM nodes.""" + +import json +import subprocess +import sys +from typing import Any + + +def is_arm() -> bool: + r = subprocess.run( + ["kubectl", "get", "nodes", "-o", "jsonpath={.items[0].status.nodeInfo.architecture}"], + capture_output=True, text=True, timeout=30, + ) + return r.returncode == 0 and r.stdout.strip() in ("arm64", "aarch64") + + +def main() -> int: + result: dict[str, Any] = {"success": False, "platform": "openshift", "smmu_detected": False} + + if not is_arm(): + result["success"] = True + result["skipped"] = True + result["info"] = "Not an ARM cluster — SMMU check skipped" + print(json.dumps(result, indent=2)) + return 0 + + node = subprocess.run( + ["kubectl", "get", "nodes", "-o", "jsonpath={.items[0].metadata.name}"], + capture_output=True, text=True, timeout=30, + ).stdout.strip() + + r = subprocess.run( + ["oc", "debug", f"node/{node}", "--", "chroot", "/host", + "bash", "-c", "dmesg | grep -i smmu | head -5"], + capture_output=True, text=True, timeout=60, + ) + smmu_output = r.stdout.strip() if r.returncode == 0 else "" + + if "smmu" in smmu_output.lower(): + result["smmu_detected"] = True + result["smmu_version"] = "v3" if "SMMUv3" in smmu_output else "detected" + result["dmesg_output"] = smmu_output[:500] + + # Also check /sys for IOMMU groups + r = subprocess.run( + ["oc", "debug", f"node/{node}", "--", "chroot", "/host", + "bash", "-c", "ls /sys/kernel/iommu_groups/ | wc -l"], + capture_output=True, text=True, timeout=60, + ) + iommu_groups = int(r.stdout.strip()) if r.returncode == 0 and r.stdout.strip().isdigit() else 0 + result["iommu_groups"] = iommu_groups + + result["success"] = result["smmu_detected"] or iommu_groups > 0 + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/arm/topology_check.py b/isvctl/configs/stubs/openshift/arm/topology_check.py new file mode 100644 index 00000000..5de87150 --- /dev/null +++ b/isvctl/configs/stubs/openshift/arm/topology_check.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Check NUMA topology on nodes.""" + +import json +import subprocess +import sys +from typing import Any + + +def is_arm() -> bool: + r = subprocess.run( + ["kubectl", "get", "nodes", "-o", "jsonpath={.items[0].status.nodeInfo.architecture}"], + capture_output=True, text=True, timeout=30, + ) + return r.returncode == 0 and r.stdout.strip() in ("arm64", "aarch64") + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "numa_nodes": 0, + } + + if not is_arm(): + result["success"] = True + result["skipped"] = True + result["info"] = "Not an ARM cluster — topology check skipped" + print(json.dumps(result, indent=2)) + return 0 + + node = subprocess.run( + ["kubectl", "get", "nodes", "-l", "nvidia.com/gpu.present=true", + "-o", "jsonpath={.items[0].metadata.name}"], + capture_output=True, text=True, timeout=30, + ).stdout.strip() + + if not node: + # Fallback to any node + node = subprocess.run( + ["kubectl", "get", "nodes", "-o", "jsonpath={.items[0].metadata.name}"], + capture_output=True, text=True, timeout=30, + ).stdout.strip() + + # Check NUMA topology + r = subprocess.run( + ["oc", "debug", f"node/{node}", "--", "chroot", "/host", + "bash", "-c", "ls -d /sys/devices/system/node/node* | wc -l"], + capture_output=True, text=True, timeout=60, + ) + numa_count = int(r.stdout.strip()) if r.returncode == 0 and r.stdout.strip().isdigit() else 0 + result["numa_nodes"] = numa_count + + # Get CPU count per NUMA node + r = subprocess.run( + ["oc", "debug", f"node/{node}", "--", "chroot", "/host", + "lscpu"], + capture_output=True, text=True, timeout=60, + ) + if r.returncode == 0: + for line in r.stdout.split("\n"): + if "NUMA node(s):" in line: + result["lscpu_numa"] = line.split(":")[-1].strip() + elif "CPU(s):" in line and "NUMA" not in line and "On-line" not in line: + result["total_cpus"] = line.split(":")[-1].strip() + elif "Model name:" in line: + result["cpu_model"] = line.split(":")[-1].strip() + + # Check GPU-NUMA affinity + r = subprocess.run( + ["oc", "debug", f"node/{node}", "--", "chroot", "/host", + "bash", "-c", "for gpu in /sys/bus/pci/devices/*/numa_node; do echo $(dirname $gpu | xargs basename):$(cat $gpu); done 2>/dev/null | grep -v ':$' | head -10"], + capture_output=True, text=True, timeout=60, + ) + if r.returncode == 0 and r.stdout.strip(): + result["pci_numa_affinity"] = r.stdout.strip().split("\n") + + result["success"] = numa_count > 0 + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/computedomain/check_crd.py b/isvctl/configs/stubs/openshift/computedomain/check_crd.py new file mode 100644 index 00000000..e41302ff --- /dev/null +++ b/isvctl/configs/stubs/openshift/computedomain/check_crd.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Check ComputeDomain CRD exists.""" + +import json +import subprocess +import sys +from typing import Any + + +def main() -> int: + result: dict[str, Any] = {"success": False, "platform": "openshift", "crd_exists": False} + + r = subprocess.run( + ["kubectl", "get", "crd", "computedomains.nvidia.com", "--no-headers"], + capture_output=True, text=True, timeout=30, + ) + result["crd_exists"] = r.returncode == 0 + + if not result["crd_exists"]: + result["error"] = "ComputeDomain CRD not found — requires GPU Operator in DRA mode" + + result["success"] = result["crd_exists"] + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/computedomain/create_domain.py b/isvctl/configs/stubs/openshift/computedomain/create_domain.py new file mode 100644 index 00000000..dd4b410a --- /dev/null +++ b/isvctl/configs/stubs/openshift/computedomain/create_domain.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Create a ComputeDomain spanning multiple GPU nodes. + +Creates a ComputeDomain CR and waits for IMEX channels to be +established between the specified nodes. + +Environment: + CD_NAMESPACE: Namespace (default: ncp-computedomain-validation) + CD_TRAY_SIZE: GPUs per tray (default: 4) +""" + +import json +import os +import subprocess +import sys +import time +from typing import Any + +NAMESPACE = os.environ.get("CD_NAMESPACE", "ncp-computedomain-validation") +DOMAIN_NAME = "ncp-test-domain" + + +def run_kubectl(*args: str, input_data: str | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, + input=input_data, timeout=120) + + +def main() -> int: + tray_size = int(os.environ.get("CD_TRAY_SIZE", "4")) + + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "domain_name": DOMAIN_NAME, + "imex_channels_ready": False, + } + + try: + run_kubectl("create", "namespace", NAMESPACE) + + # Get GPU nodes + r = run_kubectl("get", "nodes", "-l", "nvidia.com/gpu.present=true", + "-o", "jsonpath={.items[*].metadata.name}") + gpu_nodes = r.stdout.strip().split() if r.returncode == 0 and r.stdout.strip() else [] + + if len(gpu_nodes) < 2: + result["error"] = f"Need at least 2 GPU nodes, found {len(gpu_nodes)}" + print(json.dumps(result, indent=2)) + return 1 + + result["gpu_nodes"] = gpu_nodes + result["gpu_count"] = len(gpu_nodes) * tray_size + result["tray_size"] = tray_size + + # Create ComputeDomain + cd_yaml = f""" +apiVersion: nvidia.com/v1alpha1 +kind: ComputeDomain +metadata: + name: {DOMAIN_NAME} + namespace: {NAMESPACE} +spec: + gpuCount: {len(gpu_nodes) * tray_size} +""" + run_kubectl("delete", "computedomain", DOMAIN_NAME, "-n", NAMESPACE, "--ignore-not-found") + r = run_kubectl("apply", "-f", "-", input_data=cd_yaml) + if r.returncode != 0: + result["error"] = f"Failed to create ComputeDomain: {r.stderr}" + print(json.dumps(result, indent=2)) + return 1 + + # Wait for IMEX channels + print("Waiting for IMEX channels...", file=sys.stderr) + deadline = time.time() + 300 + while time.time() < deadline: + r = run_kubectl("get", "computedomain", DOMAIN_NAME, "-n", NAMESPACE, + "-o", "jsonpath={.status.phase}") + phase = r.stdout.strip() if r.returncode == 0 else "" + if phase == "Ready": + result["imex_channels_ready"] = True + break + print(f" ComputeDomain phase: {phase}", file=sys.stderr) + time.sleep(15) + + result["domain_created"] = True + result["success"] = result["imex_channels_ready"] + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/computedomain/multi_job_test.py b/isvctl/configs/stubs/openshift/computedomain/multi_job_test.py new file mode 100644 index 00000000..c180b6eb --- /dev/null +++ b/isvctl/configs/stubs/openshift/computedomain/multi_job_test.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Test multi-job scheduling within the ComputeDomain. + +Submits two GPU jobs within the same ComputeDomain and verifies +both are scheduled and complete successfully. +""" + +import json +import os +import subprocess +import sys +import time +from typing import Any + +NAMESPACE = os.environ.get("CD_NAMESPACE", "ncp-computedomain-validation") +DOMAIN_NAME = "ncp-test-domain" + + +def run_kubectl(*args: str, input_data: str | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, + input=input_data, timeout=120) + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "jobs_submitted": 0, + "jobs_completed": 0, + } + + try: + for i in range(2): + job_yaml = f""" +apiVersion: batch/v1 +kind: Job +metadata: + name: ncp-cd-job-{i} + namespace: {NAMESPACE} + labels: + nvidia.com/compute-domain: {DOMAIN_NAME} +spec: + backoffLimit: 0 + template: + metadata: + labels: + nvidia.com/compute-domain: {DOMAIN_NAME} + spec: + restartPolicy: Never + containers: + - name: gpu-job + image: nvcr.io/nvidia/cuda:12.8.0-base-ubi9 + command: ["nvidia-smi", "-L"] + resources: + limits: + nvidia.com/gpu: "1" +""" + run_kubectl("delete", "job", f"ncp-cd-job-{i}", "-n", NAMESPACE, "--ignore-not-found") + r = run_kubectl("apply", "-f", "-", input_data=job_yaml) + if r.returncode == 0: + result["jobs_submitted"] += 1 + + # Wait for jobs to complete + print("Waiting for ComputeDomain jobs...", file=sys.stderr) + deadline = time.time() + 300 + while time.time() < deadline: + completed = 0 + for i in range(2): + r = run_kubectl("get", "job", f"ncp-cd-job-{i}", "-n", NAMESPACE, + "-o", "jsonpath={.status.succeeded}") + if r.returncode == 0 and r.stdout.strip() == "1": + completed += 1 + result["jobs_completed"] = completed + if completed == 2: + break + time.sleep(10) + + result["success"] = result["jobs_completed"] == 2 + + except Exception as e: + result["error"] = str(e) + finally: + for i in range(2): + run_kubectl("delete", "job", f"ncp-cd-job-{i}", "-n", NAMESPACE, "--ignore-not-found") + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/computedomain/nccl_nvlink_test.py b/isvctl/configs/stubs/openshift/computedomain/nccl_nvlink_test.py new file mode 100644 index 00000000..52b1bab3 --- /dev/null +++ b/isvctl/configs/stubs/openshift/computedomain/nccl_nvlink_test.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""NCCL AllReduce over NVLink within the ComputeDomain. + +Runs an NCCL all_reduce_perf job using IMEX channels across +multiple nodes in the ComputeDomain. +""" + +import json +import os +import re +import subprocess +import sys +import time +from typing import Any + +NAMESPACE = os.environ.get("CD_NAMESPACE", "ncp-computedomain-validation") +DOMAIN_NAME = "ncp-test-domain" + + +def run_kubectl(*args: str, input_data: str | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, + input=input_data, timeout=120) + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "status": "failed", + "bus_bandwidth_gbps": 0.0, + } + + try: + job_yaml = f""" +apiVersion: batch/v1 +kind: Job +metadata: + name: ncp-nccl-nvlink + namespace: {NAMESPACE} + labels: + nvidia.com/compute-domain: {DOMAIN_NAME} +spec: + backoffLimit: 0 + template: + metadata: + labels: + nvidia.com/compute-domain: {DOMAIN_NAME} + spec: + restartPolicy: Never + containers: + - name: nccl + image: nvcr.io/nvidia/pytorch:24.10-py3 + command: + - bash + - -c + - | + /usr/local/bin/all_reduce_perf -b 1M -e 1G -f 2 -g $(nvidia-smi -L | wc -l) + resources: + limits: + nvidia.com/gpu: "4" +""" + run_kubectl("delete", "job", "ncp-nccl-nvlink", "-n", NAMESPACE, "--ignore-not-found") + time.sleep(2) + r = run_kubectl("apply", "-f", "-", input_data=job_yaml) + if r.returncode != 0: + result["error"] = f"Failed to create NCCL job: {r.stderr}" + print(json.dumps(result, indent=2)) + return 1 + + # Wait for completion + print("Waiting for NCCL NVLink job...", file=sys.stderr) + deadline = time.time() + 600 + while time.time() < deadline: + r = run_kubectl("get", "job", "ncp-nccl-nvlink", "-n", NAMESPACE, + "-o", "jsonpath={.status.succeeded}") + if r.returncode == 0 and r.stdout.strip() == "1": + break + r2 = run_kubectl("get", "job", "ncp-nccl-nvlink", "-n", NAMESPACE, + "-o", "jsonpath={.status.failed}") + if r2.returncode == 0 and r2.stdout.strip() == "1": + result["error"] = "NCCL job failed" + break + time.sleep(15) + + # Get logs and parse bandwidth + r = run_kubectl("logs", "-l", "job-name=ncp-nccl-nvlink", "-n", NAMESPACE) + if r.returncode == 0: + result["logs"] = r.stdout[-2000:] + # Parse peak bus bandwidth from all_reduce_perf output + bw_values = re.findall(r'(\d+\.\d+)\s*$', r.stdout, re.MULTILINE) + if bw_values: + peak_bw = max(float(v) for v in bw_values) + result["bus_bandwidth_gbps"] = peak_bw + + result["status"] = "passed" if result["bus_bandwidth_gbps"] > 0 else "failed" + result["success"] = result["status"] == "passed" + + except Exception as e: + result["error"] = str(e) + finally: + run_kubectl("delete", "job", "ncp-nccl-nvlink", "-n", NAMESPACE, "--ignore-not-found") + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/computedomain/teardown.py b/isvctl/configs/stubs/openshift/computedomain/teardown.py new file mode 100644 index 00000000..1ca3f615 --- /dev/null +++ b/isvctl/configs/stubs/openshift/computedomain/teardown.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Clean up ComputeDomain test resources.""" + +import json +import os +import subprocess +import sys +from typing import Any + +NAMESPACE = os.environ.get("CD_NAMESPACE", "ncp-computedomain-validation") +DOMAIN_NAME = "ncp-test-domain" + + +def run_kubectl(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, timeout=120) + + +def main() -> int: + result: dict[str, Any] = {"success": True, "platform": "openshift", "resources_deleted": []} + + run_kubectl("delete", "computedomain", DOMAIN_NAME, "-n", NAMESPACE, "--ignore-not-found") + result["resources_deleted"].append(f"computedomain/{DOMAIN_NAME}") + + run_kubectl("delete", "namespace", NAMESPACE, "--ignore-not-found") + result["resources_deleted"].append(f"namespace/{NAMESPACE}") + + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/computedomain/tray_allocation_test.py b/isvctl/configs/stubs/openshift/computedomain/tray_allocation_test.py new file mode 100644 index 00000000..73eb0234 --- /dev/null +++ b/isvctl/configs/stubs/openshift/computedomain/tray_allocation_test.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Test tray allocation within the ComputeDomain. + +Verifies that GPUs are allocated in tray-aligned groups and that +the scheduler respects ComputeDomain boundaries. +""" + +import json +import os +import subprocess +import sys +from typing import Any + +NAMESPACE = os.environ.get("CD_NAMESPACE", "ncp-computedomain-validation") +DOMAIN_NAME = "ncp-test-domain" + + +def run_kubectl(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, timeout=60) + + +def main() -> int: + result: dict[str, Any] = {"success": False, "platform": "openshift"} + + r = run_kubectl("get", "computedomain", DOMAIN_NAME, "-n", NAMESPACE, "-o", "json") + if r.returncode != 0: + result["error"] = f"ComputeDomain {DOMAIN_NAME} not found" + print(json.dumps(result, indent=2)) + return 1 + + cd = json.loads(r.stdout) + status = cd.get("status", {}) + result["phase"] = status.get("phase", "Unknown") + result["allocated_gpus"] = status.get("allocatedGPUs", 0) + result["available_gpus"] = status.get("availableGPUs", 0) + + # Check tray alignment + devices = status.get("devices", []) + result["device_count"] = len(devices) + result["tray_aligned"] = result["phase"] == "Ready" + + result["success"] = result["tray_aligned"] + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/dra/dra_claim_test.py b/isvctl/configs/stubs/openshift/dra/dra_claim_test.py new file mode 100644 index 00000000..8e612d13 --- /dev/null +++ b/isvctl/configs/stubs/openshift/dra/dra_claim_test.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Test ResourceClaim allocation with DRA driver. + +Creates a ResourceClaim for a GPU and a pod that references it, +verifies the claim is allocated and the pod can access the GPU. +""" + +import json +import os +import subprocess +import sys +import time +from typing import Any + +NAMESPACE = os.environ.get("K8S_NAMESPACE", "ncp-dra-validation") + + +def run_kubectl(*args: str, input_data: str | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, + input=input_data, timeout=120) + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "claim_allocated": False, + "gpu_accessible": False, + } + + try: + run_kubectl("create", "namespace", NAMESPACE) + + # Detect ResourceClass name + r = run_kubectl("get", "resourceclass", "--no-headers") + rc_name = "" + if r.returncode == 0: + for line in r.stdout.strip().split("\n"): + if "gpu" in line.lower() or "nvidia" in line.lower(): + rc_name = line.split()[0] + break + + if not rc_name: + result["error"] = "No GPU ResourceClass found" + print(json.dumps(result, indent=2)) + return 1 + + # Create ResourceClaim + Pod + manifest = f""" +apiVersion: resource.k8s.io/v1alpha3 +kind: ResourceClaim +metadata: + name: ncp-gpu-claim + namespace: {NAMESPACE} +spec: + devices: + requests: + - name: gpu + deviceClassName: {rc_name} + count: 1 +--- +apiVersion: v1 +kind: Pod +metadata: + name: ncp-dra-test + namespace: {NAMESPACE} +spec: + restartPolicy: Never + containers: + - name: gpu-test + image: nvcr.io/nvidia/cuda:12.8.0-base-ubi9 + command: ["nvidia-smi"] + resources: + claims: + - name: gpu + resourceClaims: + - name: gpu + resourceClaimName: ncp-gpu-claim +""" + run_kubectl("delete", "pod", "ncp-dra-test", "-n", NAMESPACE, "--ignore-not-found") + run_kubectl("delete", "resourceclaim", "ncp-gpu-claim", "-n", NAMESPACE, "--ignore-not-found") + time.sleep(2) + + r = run_kubectl("apply", "-f", "-", input_data=manifest) + if r.returncode != 0: + result["error"] = f"Failed to create DRA resources: {r.stderr}" + print(json.dumps(result, indent=2)) + return 1 + + # Wait for claim allocation + print("Waiting for ResourceClaim allocation...", file=sys.stderr) + deadline = time.time() + 120 + while time.time() < deadline: + r = run_kubectl("get", "resourceclaim", "ncp-gpu-claim", "-n", NAMESPACE, + "-o", "jsonpath={.status.allocation}") + if r.returncode == 0 and r.stdout.strip(): + result["claim_allocated"] = True + break + time.sleep(5) + + # Wait for pod completion + print("Waiting for GPU test pod...", file=sys.stderr) + deadline = time.time() + 180 + while time.time() < deadline: + r = run_kubectl("get", "pod", "ncp-dra-test", "-n", NAMESPACE, + "-o", "jsonpath={.status.phase}") + phase = r.stdout.strip() if r.returncode == 0 else "" + if phase == "Succeeded": + result["gpu_accessible"] = True + break + if phase == "Failed": + r = run_kubectl("logs", "ncp-dra-test", "-n", NAMESPACE) + result["error"] = f"Pod failed: {r.stdout[:200]}" + break + time.sleep(10) + + if result["gpu_accessible"]: + r = run_kubectl("logs", "ncp-dra-test", "-n", NAMESPACE) + result["nvidia_smi_output"] = r.stdout[:500] if r.returncode == 0 else "" + + result["success"] = result["claim_allocated"] and result["gpu_accessible"] + + except Exception as e: + result["error"] = str(e) + finally: + run_kubectl("delete", "pod", "ncp-dra-test", "-n", NAMESPACE, "--ignore-not-found") + run_kubectl("delete", "resourceclaim", "ncp-gpu-claim", "-n", NAMESPACE, "--ignore-not-found") + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/dra/dra_scc_check.py b/isvctl/configs/stubs/openshift/dra/dra_scc_check.py new file mode 100644 index 00000000..156567aa --- /dev/null +++ b/isvctl/configs/stubs/openshift/dra/dra_scc_check.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Check SCC compatibility with DRA driver pods. + +Verifies that the DRA driver service account has appropriate SCC +privileges and that DRA pods are not blocked by SCC restrictions. +""" + +import json +import subprocess +import sys +from typing import Any + +GPU_NS_CANDIDATES = ["nvidia-gpu-operator", "gpu-operator", "openshift-operators"] + + +def run_kubectl(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, timeout=60) + + +def run_oc(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["oc"] + list(args), capture_output=True, text=True, timeout=60) + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "scc_compatible": False, + "dra_pods_healthy": False, + } + + # Find GPU operator namespace with DRA pods + gpu_ns = "" + for ns in GPU_NS_CANDIDATES: + r = run_kubectl("get", "pods", "-n", ns, "--no-headers") + if r.returncode == 0 and "dra" in r.stdout.lower(): + gpu_ns = ns + break + + if not gpu_ns: + result["error"] = "DRA driver pods not found" + print(json.dumps(result, indent=2)) + return 1 + + # Check DRA pods are not in CrashLoopBackOff or Error + r = run_kubectl("get", "pods", "-n", gpu_ns, "--no-headers") + dra_pods = [l for l in r.stdout.strip().split("\n") + if "dra" in l.lower() or "nvidia-dra" in l.lower()] + + unhealthy = [l for l in dra_pods if "CrashLoop" in l or "Error" in l or "CreateContainerConfigError" in l] + result["dra_pods_healthy"] = len(unhealthy) == 0 + result["dra_pod_count"] = len(dra_pods) + result["unhealthy_pods"] = len(unhealthy) + + # Check SCC assigned to DRA pods + scc_set = set() + for line in dra_pods: + pod_name = line.split()[0] + r = run_oc("get", "pod", pod_name, "-n", gpu_ns, + "-o", "jsonpath={.metadata.annotations.openshift\\.io/scc}") + if r.returncode == 0 and r.stdout.strip(): + scc_set.add(r.stdout.strip()) + + result["scc_assigned"] = sorted(scc_set) + + # SCC is compatible if DRA pods are healthy (not blocked by SCC) + result["scc_compatible"] = result["dra_pods_healthy"] + result["success"] = result["scc_compatible"] + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/dra/dra_setup.py b/isvctl/configs/stubs/openshift/dra/dra_setup.py new file mode 100644 index 00000000..ab10c0db --- /dev/null +++ b/isvctl/configs/stubs/openshift/dra/dra_setup.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Verify GPU Operator DRA driver is healthy. + +Checks that the DRA driver DaemonSet pods are running and +the ResourceClass for GPUs exists. +""" + +import json +import subprocess +import sys +from typing import Any + +GPU_NS_CANDIDATES = ["nvidia-gpu-operator", "gpu-operator", "openshift-operators"] + + +def run_kubectl(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, timeout=60) + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "driver_ready": False, + "resource_class_exists": False, + "pods_ready": 0, + "pods_desired": 0, + } + + # Find DRA driver pods + gpu_ns = "" + for ns in GPU_NS_CANDIDATES: + r = run_kubectl("get", "pods", "-n", ns, "--no-headers") + if r.returncode == 0 and "dra-driver" in r.stdout.lower(): + gpu_ns = ns + break + + if not gpu_ns: + result["error"] = "DRA driver pods not found — is GPU Operator in DRA mode?" + print(json.dumps(result, indent=2)) + return 1 + + result["namespace"] = gpu_ns + + # Count DRA driver pods + r = run_kubectl("get", "pods", "-n", gpu_ns, "--no-headers") + dra_pods = [l for l in r.stdout.strip().split("\n") + if "dra-driver" in l.lower() or "nvidia-dra" in l.lower()] + running = sum(1 for l in dra_pods if "Running" in l) + result["pods_ready"] = running + result["pods_desired"] = len(dra_pods) + result["driver_ready"] = running > 0 + + # Check ResourceClass + r = run_kubectl("get", "resourceclass", "--no-headers") + if r.returncode == 0: + for line in r.stdout.strip().split("\n"): + if "gpu" in line.lower() or "nvidia" in line.lower(): + result["resource_class_exists"] = True + result["resource_class_name"] = line.split()[0] + break + + # Check DRA feature gate + r = run_kubectl("get", "featuregate", "cluster", "-o", + "jsonpath={.spec.featureSet}") + result["feature_set"] = r.stdout.strip() if r.returncode == 0 else "Unknown" + + result["success"] = result["driver_ready"] and result["resource_class_exists"] + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/dra/teardown.py b/isvctl/configs/stubs/openshift/dra/teardown.py new file mode 100644 index 00000000..b25b4263 --- /dev/null +++ b/isvctl/configs/stubs/openshift/dra/teardown.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Clean up DRA test resources.""" + +import json +import os +import subprocess +import sys +from typing import Any + +NAMESPACE = os.environ.get("K8S_NAMESPACE", "ncp-dra-validation") + + +def main() -> int: + result: dict[str, Any] = {"success": True, "platform": "openshift", "resources_deleted": []} + + subprocess.run(["kubectl", "delete", "namespace", NAMESPACE, "--ignore-not-found"], + capture_output=True, text=True, timeout=60) + result["resources_deleted"].append(f"namespace/{NAMESPACE}") + + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/gpu-health/dcgm_exporter_check.py b/isvctl/configs/stubs/openshift/gpu-health/dcgm_exporter_check.py new file mode 100644 index 00000000..6a5b70bb --- /dev/null +++ b/isvctl/configs/stubs/openshift/gpu-health/dcgm_exporter_check.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Check DCGM Exporter is running and exposing metrics.""" + +import json +import subprocess +import sys +from typing import Any + +GPU_NS_CANDIDATES = ["nvidia-gpu-operator", "gpu-operator", "openshift-operators"] + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "exporter_running": False, + "metrics_endpoint": "", + "pod_count": 0, + } + + # Find GPU operator namespace + gpu_ns = "" + for ns in GPU_NS_CANDIDATES: + r = subprocess.run(["kubectl", "get", "pods", "-n", ns, "-l", + "app=nvidia-dcgm-exporter", "--no-headers"], + capture_output=True, text=True, timeout=30) + if r.returncode == 0 and r.stdout.strip(): + gpu_ns = ns + break + + if not gpu_ns: + # Try broader label + for ns in GPU_NS_CANDIDATES: + r = subprocess.run(["kubectl", "get", "pods", "-n", ns, "--no-headers"], + capture_output=True, text=True, timeout=30) + if r.returncode == 0 and "dcgm" in r.stdout.lower(): + gpu_ns = ns + break + + if not gpu_ns: + result["error"] = "DCGM Exporter pods not found in any known namespace" + print(json.dumps(result, indent=2)) + return 1 + + result["namespace"] = gpu_ns + + # Count running DCGM exporter pods + r = subprocess.run(["kubectl", "get", "pods", "-n", gpu_ns, "--no-headers"], + capture_output=True, text=True, timeout=30) + dcgm_pods = [l for l in r.stdout.strip().split("\n") + if "dcgm" in l.lower() and "Running" in l] + result["pod_count"] = len(dcgm_pods) + result["exporter_running"] = len(dcgm_pods) > 0 + + # Check metrics service + r = subprocess.run(["kubectl", "get", "svc", "-n", gpu_ns, "-l", + "app=nvidia-dcgm-exporter", + "-o", "jsonpath={.items[0].metadata.name}"], + capture_output=True, text=True, timeout=30) + svc_name = r.stdout.strip() if r.returncode == 0 else "" + if svc_name: + result["metrics_endpoint"] = f"{svc_name}.{gpu_ns}.svc:9400/metrics" + + # Check ServiceMonitor exists (for Prometheus scraping) + r = subprocess.run(["kubectl", "get", "servicemonitor", "-n", gpu_ns, + "--no-headers"], + capture_output=True, text=True, timeout=30) + result["service_monitor_exists"] = r.returncode == 0 and "dcgm" in r.stdout.lower() + + result["success"] = result["exporter_running"] + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/gpu-health/dcgm_metrics_validate.py b/isvctl/configs/stubs/openshift/gpu-health/dcgm_metrics_validate.py new file mode 100644 index 00000000..62405e82 --- /dev/null +++ b/isvctl/configs/stubs/openshift/gpu-health/dcgm_metrics_validate.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Validate DCGM metrics content by scraping a DCGM exporter pod.""" + +import json +import subprocess +import sys +from typing import Any + +GPU_NS_CANDIDATES = ["nvidia-gpu-operator", "gpu-operator", "openshift-operators"] +EXPECTED_METRICS = [ + "DCGM_FI_DEV_GPU_UTIL", + "DCGM_FI_DEV_MEM_COPY_UTIL", + "DCGM_FI_DEV_GPU_TEMP", + "DCGM_FI_DEV_POWER_USAGE", + "DCGM_FI_DEV_FB_USED", + "DCGM_FI_DEV_FB_FREE", +] + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "metrics_found": [], + "metrics_missing": [], + } + + # Find a DCGM exporter pod + dcgm_pod = "" + gpu_ns = "" + for ns in GPU_NS_CANDIDATES: + r = subprocess.run(["kubectl", "get", "pods", "-n", ns, "--no-headers"], + capture_output=True, text=True, timeout=30) + if r.returncode == 0: + for line in r.stdout.strip().split("\n"): + if "dcgm" in line.lower() and "Running" in line: + dcgm_pod = line.split()[0] + gpu_ns = ns + break + if dcgm_pod: + break + + if not dcgm_pod: + result["error"] = "No running DCGM exporter pod found" + print(json.dumps(result, indent=2)) + return 1 + + # Scrape metrics from the pod + r = subprocess.run( + ["kubectl", "exec", dcgm_pod, "-n", gpu_ns, "--", + "curl", "-s", "http://localhost:9400/metrics"], + capture_output=True, text=True, timeout=30, + ) + if r.returncode != 0: + # Fallback: port-forward and curl + r = subprocess.run( + ["kubectl", "exec", dcgm_pod, "-n", gpu_ns, "--", + "wget", "-qO-", "http://localhost:9400/metrics"], + capture_output=True, text=True, timeout=30, + ) + + metrics_text = r.stdout if r.returncode == 0 else "" + + for metric in EXPECTED_METRICS: + if metric in metrics_text: + result["metrics_found"].append(metric) + else: + result["metrics_missing"].append(metric) + + result["total_lines"] = len(metrics_text.strip().split("\n")) if metrics_text else 0 + result["success"] = len(result["metrics_found"]) > 0 + + if result["metrics_missing"]: + result["warning"] = f"Missing metrics: {', '.join(result['metrics_missing'])}" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/gpu-health/nhc_operator_check.py b/isvctl/configs/stubs/openshift/gpu-health/nhc_operator_check.py new file mode 100644 index 00000000..5b71d286 --- /dev/null +++ b/isvctl/configs/stubs/openshift/gpu-health/nhc_operator_check.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Check Node Health Check (NHC) operator status.""" + +import json +import subprocess +import sys +from typing import Any + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "nhc_installed": False, + "nhc_configs": 0, + } + + # Check for NodeHealthCheck CRD + r = subprocess.run( + ["kubectl", "get", "crd", "nodehealthchecks.remediation.medik8s.io", + "--no-headers"], + capture_output=True, text=True, timeout=30, + ) + if r.returncode != 0: + result["success"] = True + result["info"] = "NHC operator not installed — skipping" + print(json.dumps(result, indent=2)) + return 0 + + result["nhc_installed"] = True + + # Check NodeHealthCheck resources + r = subprocess.run( + ["kubectl", "get", "nodehealthcheck", "--all-namespaces", "-o", "json"], + capture_output=True, text=True, timeout=30, + ) + if r.returncode == 0: + data = json.loads(r.stdout) + result["nhc_configs"] = len(data.get("items", [])) + for item in data.get("items", []): + name = item["metadata"]["name"] + phase = item.get("status", {}).get("phase", "Unknown") + result.setdefault("configs", []).append({"name": name, "phase": phase}) + + # Check for MachineHealthCheck (OpenShift native alternative) + r = subprocess.run( + ["kubectl", "get", "machinehealthcheck", "-n", "openshift-machine-api", + "--no-headers"], + capture_output=True, text=True, timeout=30, + ) + if r.returncode == 0 and r.stdout.strip(): + mhc_count = len(r.stdout.strip().split("\n")) + result["machine_health_checks"] = mhc_count + + result["success"] = True + + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/gpu-health/prometheus_gpu_metrics.py b/isvctl/configs/stubs/openshift/gpu-health/prometheus_gpu_metrics.py new file mode 100644 index 00000000..de683606 --- /dev/null +++ b/isvctl/configs/stubs/openshift/gpu-health/prometheus_gpu_metrics.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Verify Prometheus is scraping GPU metrics. + +Queries the OpenShift Thanos/Prometheus endpoint for GPU metrics +to confirm the monitoring pipeline works end-to-end. +""" + +import json +import subprocess +import sys +from typing import Any + +METRIC_QUERIES = [ + "DCGM_FI_DEV_GPU_UTIL", + "DCGM_FI_DEV_GPU_TEMP", + "DCGM_FI_DEV_POWER_USAGE", +] + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "metrics_in_prometheus": [], + "metrics_missing": [], + } + + # Get Thanos querier route + r = subprocess.run( + ["oc", "get", "route", "thanos-querier", "-n", "openshift-monitoring", + "-o", "jsonpath={.spec.host}"], + capture_output=True, text=True, timeout=30, + ) + if r.returncode != 0 or not r.stdout.strip(): + result["error"] = "Thanos querier route not found" + print(json.dumps(result, indent=2)) + return 1 + + thanos_host = r.stdout.strip() + + # Get bearer token for Prometheus access + r = subprocess.run(["oc", "whoami", "-t"], capture_output=True, text=True, timeout=10) + token = r.stdout.strip() if r.returncode == 0 else "" + + if not token: + result["error"] = "Could not get bearer token for Prometheus" + print(json.dumps(result, indent=2)) + return 1 + + # Query each metric + for metric in METRIC_QUERIES: + r = subprocess.run( + ["curl", "-sk", + f"https://{thanos_host}/api/v1/query?query={metric}", + "-H", f"Authorization: Bearer {token}"], + capture_output=True, text=True, timeout=30, + ) + if r.returncode == 0: + try: + data = json.loads(r.stdout) + if data.get("status") == "success": + results = data.get("data", {}).get("result", []) + if results: + result["metrics_in_prometheus"].append(metric) + continue + except json.JSONDecodeError: + pass + result["metrics_missing"].append(metric) + + result["success"] = len(result["metrics_in_prometheus"]) > 0 + + if result["metrics_missing"]: + result["warning"] = f"Metrics not in Prometheus: {', '.join(result['metrics_missing'])}" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/hosted/configure_hosted_cluster.py b/isvctl/configs/stubs/openshift/hosted/configure_hosted_cluster.py new file mode 100644 index 00000000..41c23db8 --- /dev/null +++ b/isvctl/configs/stubs/openshift/hosted/configure_hosted_cluster.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Configure day-2 operators on the hosted cluster. + +Targets the hosted cluster's kubeconfig to install: +- GPU Operator +- Network Operator +- OpenShift Virtualization (optional) + +Environment: + HOSTED_STATE_FILE: Path to state file (default: /tmp/ncp-hosted-state.json) +""" + +import json +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +STATE_FILE = os.environ.get("HOSTED_STATE_FILE", "/tmp/ncp-hosted-state.json") + + +def load_state() -> dict[str, Any]: + try: + return json.loads(Path(STATE_FILE).read_text()) + except (FileNotFoundError, json.JSONDecodeError): + return {} + + +def run_hosted_oc(*args: str, kubeconfig: str, input_data: str | None = None) -> subprocess.CompletedProcess[str]: + env = {**os.environ, "KUBECONFIG": kubeconfig} + return subprocess.run(["oc"] + list(args), capture_output=True, text=True, + input=input_data, timeout=120, env=env) + + +def main() -> int: + state = load_state() + kubeconfig = state.get("hosted_kubeconfig", "") + + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "gpu_operator_installed": False, + "network_operator_installed": False, + } + + if not kubeconfig or not Path(kubeconfig).exists(): + result["error"] = f"Hosted kubeconfig not found at {kubeconfig}" + print(json.dumps(result, indent=2)) + return 1 + + try: + # Verify connectivity to hosted cluster + r = run_hosted_oc("whoami", kubeconfig=kubeconfig) + if r.returncode != 0: + result["error"] = f"Cannot connect to hosted cluster: {r.stderr}" + print(json.dumps(result, indent=2)) + return 1 + + print(f"Connected to hosted cluster as {r.stdout.strip()}", file=sys.stderr) + + # Install GPU Operator + print("Installing GPU Operator on hosted cluster...", file=sys.stderr) + run_hosted_oc("create", "namespace", "nvidia-gpu-operator", kubeconfig=kubeconfig) + + og_yaml = """ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: nvidia-gpu-operator + namespace: nvidia-gpu-operator +spec: + targetNamespaces: + - nvidia-gpu-operator +""" + run_hosted_oc("apply", "-f", "-", kubeconfig=kubeconfig, input_data=og_yaml) + + sub_yaml = """ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: gpu-operator-certified + namespace: nvidia-gpu-operator +spec: + channel: "v24.9" + installPlanApproval: Automatic + name: gpu-operator-certified + source: certified-operators + sourceNamespace: openshift-marketplace +""" + run_hosted_oc("apply", "-f", "-", kubeconfig=kubeconfig, input_data=sub_yaml) + + # Wait for GPU Operator + deadline = time.time() + 300 + while time.time() < deadline: + r = run_hosted_oc("get", "pods", "-n", "nvidia-gpu-operator", + "-l", "app=gpu-operator", "--no-headers", + kubeconfig=kubeconfig) + if r.returncode == 0 and "Running" in r.stdout: + result["gpu_operator_installed"] = True + break + time.sleep(15) + + # Install Network Operator + print("Installing Network Operator on hosted cluster...", file=sys.stderr) + run_hosted_oc("create", "namespace", "nvidia-network-operator", kubeconfig=kubeconfig) + + net_sub_yaml = """ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: nvidia-network-operator + namespace: nvidia-network-operator +spec: + channel: "v24.7" + installPlanApproval: Automatic + name: nvidia-network-operator-certified + source: certified-operators + sourceNamespace: openshift-marketplace +""" + run_hosted_oc("apply", "-f", "-", kubeconfig=kubeconfig, input_data=net_sub_yaml) + result["network_operator_installed"] = True # Best effort + + result["success"] = result["gpu_operator_installed"] + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/hosted/create_hosted_realm.py b/isvctl/configs/stubs/openshift/hosted/create_hosted_realm.py new file mode 100644 index 00000000..c3c7afe1 --- /dev/null +++ b/isvctl/configs/stubs/openshift/hosted/create_hosted_realm.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Create a Keycloak realm for the hosted cluster. + +Uses the Keycloak instance running on the management cluster +(deployed by iam.yaml) to create a separate realm and OIDC client +for the hosted cluster's OAuth configuration. + +Environment: + KEYCLOAK_NAMESPACE: Keycloak namespace on mgmt cluster (default: ncp-keycloak) + HOSTED_CLUSTER_NAME: Hosted cluster name (default: ncp-hosted) +""" + +import base64 +import json +import os +import ssl +import subprocess +import sys +import urllib.error +import urllib.parse +import urllib.request +from typing import Any + + +def run_oc(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["oc"] + list(args), capture_output=True, text=True, timeout=60) + + +def main() -> int: + namespace = os.environ.get("KEYCLOAK_NAMESPACE", "ncp-keycloak") + cluster_name = os.environ.get("HOSTED_CLUSTER_NAME", "ncp-hosted") + realm_name = f"{cluster_name}" + + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "realm_name": realm_name, + "realm_created": False, + } + + try: + # Get Keycloak route + r = run_oc("get", "routes", "-n", namespace, "-o", "jsonpath={.items[0].spec.host}") + if r.returncode != 0 or not r.stdout.strip(): + result["error"] = "Keycloak not found. Run iam.yaml first." + print(json.dumps(result, indent=2)) + return 1 + keycloak_url = f"https://{r.stdout.strip()}" + + # Get admin credentials + for secret_name in ["ncp-keycloak-initial-admin", "keycloak-initial-admin"]: + r = run_oc("get", "secret", secret_name, "-n", namespace, + "-o", "jsonpath={.data.username}") + if r.returncode == 0 and r.stdout.strip(): + admin_user = base64.b64decode(r.stdout.strip()).decode() + r2 = run_oc("get", "secret", secret_name, "-n", namespace, + "-o", "jsonpath={.data.password}") + admin_pass = base64.b64decode(r2.stdout.strip()).decode() + break + else: + result["error"] = "Could not get Keycloak admin credentials" + print(json.dumps(result, indent=2)) + return 1 + + # Get admin token + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + token_url = f"{keycloak_url}/realms/master/protocol/openid-connect/token" + body = urllib.parse.urlencode({ + "grant_type": "password", "client_id": "admin-cli", + "username": admin_user, "password": admin_pass, + }).encode() + req = urllib.request.Request(token_url, data=body, method="POST") + req.add_header("Content-Type", "application/x-www-form-urlencoded") + resp = urllib.request.urlopen(req, context=ctx, timeout=30) + token = json.loads(resp.read().decode())["access_token"] + + # Create realm (idempotent) + realm_data = json.dumps({"realm": realm_name, "enabled": True}).encode() + req = urllib.request.Request( + f"{keycloak_url}/admin/realms", data=realm_data, method="POST") + req.add_header("Authorization", f"Bearer {token}") + req.add_header("Content-Type", "application/json") + try: + urllib.request.urlopen(req, context=ctx, timeout=30) + result["realm_created"] = True + except urllib.error.HTTPError as e: + if e.code == 409: + result["realm_created"] = True # Already exists + else: + raise + + result["keycloak_url"] = keycloak_url + result["success"] = result["realm_created"] + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/hosted/deploy_capi_provider.py b/isvctl/configs/stubs/openshift/hosted/deploy_capi_provider.py new file mode 100644 index 00000000..6b1ed868 --- /dev/null +++ b/isvctl/configs/stubs/openshift/hosted/deploy_capi_provider.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Deploy Cluster API Provider for Carbide on management cluster. + +Installs the CAPI core components and the Carbide infrastructure +provider. Idempotent — safe to run repeatedly. +""" + +import json +import os +import subprocess +import sys +import time +from typing import Any + + +def run_kubectl(*args: str, input_data: str | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, + input=input_data, timeout=120) + + +def run_oc(*args: str, input_data: str | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run(["oc"] + list(args), capture_output=True, text=True, + input=input_data, timeout=120) + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "capi_installed": False, + "carbide_provider_installed": False, + } + + try: + # Check if CAPI is already installed + r = run_kubectl("get", "crd", "clusters.cluster.x-k8s.io", "--no-headers") + if r.returncode == 0: + print("CAPI CRDs already installed.", file=sys.stderr) + result["capi_installed"] = True + else: + # Install CAPI via clusterctl + r = subprocess.run( + ["clusterctl", "init", "--infrastructure", "carbide"], + capture_output=True, text=True, timeout=300, + ) + if r.returncode != 0: + # Fallback: install via manifests + print("clusterctl not available, installing CAPI via manifests...", file=sys.stderr) + + # Install CAPI core + capi_ns = "capi-system" + run_kubectl("create", "namespace", capi_ns) + + # Apply CAPI operator subscription + sub_yaml = f""" +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: cluster-api-operator + namespace: {capi_ns} +spec: + channel: stable + name: cluster-api-operator + source: redhat-operators + sourceNamespace: openshift-marketplace + installPlanApproval: Automatic +""" + run_oc("apply", "-f", "-", input_data=sub_yaml) + + # Wait for CAPI CRDs + print("Waiting for CAPI CRDs...", file=sys.stderr) + deadline = time.time() + 300 + while time.time() < deadline: + r = run_kubectl("get", "crd", "clusters.cluster.x-k8s.io", "--no-headers") + if r.returncode == 0: + result["capi_installed"] = True + break + time.sleep(10) + else: + result["capi_installed"] = True + + # Check Carbide infrastructure provider + r = run_kubectl("get", "crd", "carbidemachines.infrastructure.cluster.x-k8s.io", + "--no-headers") + if r.returncode == 0: + print("Carbide CAPI provider already installed.", file=sys.stderr) + result["carbide_provider_installed"] = True + else: + # Install Carbide infrastructure provider + carbide_provider_ns = "capc-system" + run_kubectl("create", "namespace", carbide_provider_ns) + + # Apply Carbide provider manifests + # The provider is installed via InfrastructureProvider CR + ip_yaml = f""" +apiVersion: operator.cluster.x-k8s.io/v1alpha2 +kind: InfrastructureProvider +metadata: + name: carbide + namespace: {carbide_provider_ns} +spec: + version: v0.1.0 + fetchConfig: + url: https://github.com/fabiendupont/cluster-api-provider-nvidia-carbide/releases +""" + run_oc("apply", "-f", "-", input_data=ip_yaml) + + # Wait for Carbide provider CRDs + print("Waiting for Carbide CAPI provider...", file=sys.stderr) + deadline = time.time() + 300 + while time.time() < deadline: + r = run_kubectl("get", "deployment", "-n", carbide_provider_ns, "--no-headers") + if r.returncode == 0 and r.stdout.strip(): + result["carbide_provider_installed"] = True + break + time.sleep(10) + + result["success"] = result["capi_installed"] + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/hosted/deploy_hosted_cluster.py b/isvctl/configs/stubs/openshift/hosted/deploy_hosted_cluster.py new file mode 100644 index 00000000..9964df30 --- /dev/null +++ b/isvctl/configs/stubs/openshift/hosted/deploy_hosted_cluster.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Deploy the hosted OpenShift cluster via CAPI. + +Creates a CAPI Cluster CR referencing the Carbide infrastructure +provider and the BareMetalHosts provisioned in the previous step. +Waits for the hosted cluster to be fully operational. + +Environment: + HOSTED_CLUSTER_NAME: Cluster name (default: ncp-hosted) + HOSTED_BASE_DOMAIN: Base DNS domain (default: from mgmt cluster) + OCP_PULL_SECRET: OpenShift pull secret (JSON or file path) +""" + +import json +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +STATE_FILE = os.environ.get("HOSTED_STATE_FILE", "/tmp/ncp-hosted-state.json") + + +def run_kubectl(*args: str, input_data: str | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, + input=input_data, timeout=120) + + +def run_oc(*args: str, input_data: str | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run(["oc"] + list(args), capture_output=True, text=True, + input=input_data, timeout=120) + + +def load_state() -> dict[str, Any]: + try: + return json.loads(Path(STATE_FILE).read_text()) + except (FileNotFoundError, json.JSONDecodeError): + return {} + + +def save_state(state: dict[str, Any]) -> None: + Path(STATE_FILE).write_text(json.dumps(state, indent=2)) + + +def main() -> int: + cluster_name = os.environ.get("HOSTED_CLUSTER_NAME", "ncp-hosted") + pull_secret = os.environ.get("OCP_PULL_SECRET", "") + + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "cluster_ready": False, + "kubeconfig_path": "", + } + + try: + state = load_state() + + # Get base domain from management cluster + base_domain = os.environ.get("HOSTED_BASE_DOMAIN", "") + if not base_domain: + r = run_oc("get", "ingress.config.openshift.io", "cluster", + "-o", "jsonpath={.spec.domain}") + if r.returncode == 0 and r.stdout.strip(): + # apps.cluster.example.com → cluster.example.com + base_domain = r.stdout.strip().replace("apps.", "", 1) + + cluster_ns = f"{cluster_name}" + run_kubectl("create", "namespace", cluster_ns) + + # Create pull secret in cluster namespace + if pull_secret: + ps_content = pull_secret + if os.path.isfile(pull_secret): + ps_content = Path(pull_secret).read_text() + + ps_yaml = f""" +apiVersion: v1 +kind: Secret +metadata: + name: pull-secret + namespace: {cluster_ns} +type: kubernetes.io/dockerconfigjson +stringData: + .dockerconfigjson: '{ps_content}' +""" + run_kubectl("apply", "-f", "-", input_data=ps_yaml) + + # Create CAPI Cluster CR + worker_count = len(state.get("bmh_instance_ids", [])) + cluster_yaml = f""" +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: {cluster_name} + namespace: {cluster_ns} +spec: + clusterNetwork: + pods: + cidrBlocks: ["10.132.0.0/14"] + services: + cidrBlocks: ["172.31.0.0/16"] + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: HostedControlPlane + name: {cluster_name} + namespace: {cluster_ns} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: CarbideCluster + name: {cluster_name} + namespace: {cluster_ns} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: CarbideCluster +metadata: + name: {cluster_name} + namespace: {cluster_ns} +spec: + controlPlaneEndpoint: + host: api.{cluster_name}.{base_domain} + port: 6443 +""" + r = run_kubectl("apply", "-f", "-", input_data=cluster_yaml) + if r.returncode != 0: + result["error"] = f"Failed to create Cluster CR: {r.stderr}" + print(json.dumps(result, indent=2)) + return 1 + + # Wait for cluster to be provisioned + print("Waiting for hosted cluster to be ready...", file=sys.stderr) + kubeconfig_path = f"/tmp/{cluster_name}-kubeconfig" + + deadline = time.time() + 5400 # 90 minutes + while time.time() < deadline: + r = run_kubectl("get", "cluster", cluster_name, "-n", cluster_ns, + "-o", "jsonpath={.status.phase}") + phase = r.stdout.strip() if r.returncode == 0 else "" + + if phase == "Provisioned": + result["cluster_ready"] = True + break + + print(f" Cluster phase: {phase}", file=sys.stderr) + time.sleep(30) + + # Extract kubeconfig + r = run_kubectl("get", "secret", f"{cluster_name}-kubeconfig", "-n", cluster_ns, + "-o", "jsonpath={.data.value}") + if r.returncode == 0 and r.stdout.strip(): + import base64 + kubeconfig = base64.b64decode(r.stdout.strip()).decode() + Path(kubeconfig_path).write_text(kubeconfig) + result["kubeconfig_path"] = kubeconfig_path + + state["hosted_kubeconfig"] = kubeconfig_path + state["hosted_cluster_ns"] = cluster_ns + state["hosted_base_domain"] = base_domain + save_state(state) + + result["success"] = result["cluster_ready"] + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/hosted/provision_baremetal_hosts.py b/isvctl/configs/stubs/openshift/hosted/provision_baremetal_hosts.py new file mode 100644 index 00000000..48ed3553 --- /dev/null +++ b/isvctl/configs/stubs/openshift/hosted/provision_baremetal_hosts.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Provision BareMetalHosts via Carbide for the hosted cluster. + +Creates bare-metal instances in Carbide and registers them as +BareMetalHost resources in the management cluster for CAPI to use. + +Environment: + CARBIDE_SITE_ID: Site UUID + CARBIDE_INSTANCE_TYPE: Instance type for GPU workers + HOSTED_WORKER_COUNT: Number of workers (default: 6) + HOSTED_CLUSTER_NAME: Cluster name prefix (default: ncp-hosted) +""" + +import json +import os +import subprocess +import sys +import time +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "stubs")) +try: + from carbide.common.carbide import run_carbide, load_state, save_state +except ImportError: + def run_carbide(*args: str, timeout: int = 600) -> dict[str, Any]: + cmd = ["carbidecli", "-o", "json"] + list(args) + r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + if r.returncode != 0: + raise RuntimeError(f"carbidecli failed: {r.stderr}") + return json.loads(r.stdout) + + def load_state(sf: str | None = None) -> dict[str, Any]: + p = sf or "/tmp/ncp-hosted-state.json" + try: + return json.loads(open(p).read()) + except (FileNotFoundError, json.JSONDecodeError): + return {} + + def save_state(s: dict[str, Any], sf: str | None = None) -> None: + p = sf or "/tmp/ncp-hosted-state.json" + open(p, "w").write(json.dumps(s, indent=2)) + + +STATE_FILE = os.environ.get("HOSTED_STATE_FILE", "/tmp/ncp-hosted-state.json") + + +def run_kubectl(*args: str, input_data: str | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, + input=input_data, timeout=120) + + +def main() -> int: + site_id = os.environ.get("CARBIDE_SITE_ID", "") + instance_type = os.environ.get("CARBIDE_INSTANCE_TYPE", "") + worker_count = int(os.environ.get("HOSTED_WORKER_COUNT", "6")) + cluster_name = os.environ.get("HOSTED_CLUSTER_NAME", "ncp-hosted") + + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "hosts_ready": 0, + "host_count": worker_count, + } + + if not site_id or not instance_type: + result["error"] = "CARBIDE_SITE_ID and CARBIDE_INSTANCE_TYPE are required" + print(json.dumps(result, indent=2)) + return 1 + + try: + state = load_state(STATE_FILE) + + if state.get("bmh_instance_ids"): + print("BareMetalHosts already provisioned.", file=sys.stderr) + result["hosts_ready"] = len(state["bmh_instance_ids"]) + result["success"] = result["hosts_ready"] >= worker_count + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + # Batch-create instances in Carbide + print(f"Provisioning {worker_count} bare-metal instances...", file=sys.stderr) + batch_result = run_carbide( + "instance", "batch-create", + "--name-prefix", f"{cluster_name}-worker", + "--count", str(worker_count), + "--site-id", site_id, + "--instance-type", instance_type, + ) + instances = batch_result if isinstance(batch_result, list) else batch_result.get("instances", [batch_result]) + instance_ids = [i.get("id", "") for i in instances] + + state["bmh_instance_ids"] = instance_ids + state["hosted_cluster_name"] = cluster_name + save_state(state, STATE_FILE) + + # Wait for instances and collect MAC addresses + print("Waiting for instances to be ready...", file=sys.stderr) + bmh_data = [] + for iid in instance_ids: + deadline = time.time() + 1800 + while time.time() < deadline: + info = run_carbide("instance", "get", iid) + status = info.get("status", info.get("state", "")).lower() + if status in ("running", "active", "ready"): + bmh_data.append({ + "instance_id": iid, + "mac": info.get("macAddress", info.get("mac_address", "")), + "ip": info.get("ip_address", info.get("public_ip", "")), + "name": info.get("name", iid), + }) + break + time.sleep(30) + + state["bmh_data"] = bmh_data + save_state(state, STATE_FILE) + + # Create BareMetalHost resources in management cluster + bmh_ns = f"{cluster_name}-hosts" + run_kubectl("create", "namespace", bmh_ns) + + for bmh in bmh_data: + bmh_yaml = f""" +apiVersion: metal3.io/v1alpha1 +kind: BareMetalHost +metadata: + name: {bmh['name']} + namespace: {bmh_ns} + labels: + cluster.x-k8s.io/cluster-name: {cluster_name} +spec: + online: true + bootMACAddress: "{bmh['mac']}" + bmc: + address: "carbide://{bmh['instance_id']}" + credentialsName: carbide-bmc-secret +""" + run_kubectl("apply", "-f", "-", input_data=bmh_yaml) + + result["hosts_ready"] = len(bmh_data) + result["bmh_namespace"] = bmh_ns + result["success"] = result["hosts_ready"] >= worker_count + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/hosted/teardown.py b/isvctl/configs/stubs/openshift/hosted/teardown.py new file mode 100644 index 00000000..20e0cc32 --- /dev/null +++ b/isvctl/configs/stubs/openshift/hosted/teardown.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Tear down the hosted cluster and BareMetalHosts. + +Deletes the CAPI Cluster CR (which triggers hosted cluster deletion), +then deletes BareMetalHost resources and terminates Carbide instances. + +Gated by TEARDOWN_ENABLED=true to prevent accidental deletion. + +Environment: + TEARDOWN_ENABLED: Must be "true" to delete resources + HOSTED_STATE_FILE: Path to state file (default: /tmp/ncp-hosted-state.json) +""" + +import json +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +STATE_FILE = os.environ.get("HOSTED_STATE_FILE", "/tmp/ncp-hosted-state.json") + + +def load_state() -> dict[str, Any]: + try: + return json.loads(Path(STATE_FILE).read_text()) + except (FileNotFoundError, json.JSONDecodeError): + return {} + + +def run_kubectl(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, timeout=120) + + +def run_carbide(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["carbidecli", "-o", "json"] + list(args), + capture_output=True, text=True, timeout=600) + + +def main() -> int: + teardown_enabled = os.environ.get("TEARDOWN_ENABLED", "false").lower() == "true" + + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "resources_deleted": [], + } + + if not teardown_enabled: + result["success"] = True + result["skipped"] = True + result["message"] = "Hosted cluster preserved (TEARDOWN_ENABLED != true)" + print(json.dumps(result, indent=2)) + return 0 + + state = load_state() + cluster_name = state.get("hosted_cluster_name", "ncp-hosted") + cluster_ns = state.get("hosted_cluster_ns", cluster_name) + errors = [] + + # Delete CAPI Cluster (triggers hosted cluster deletion) + print("Deleting hosted cluster...", file=sys.stderr) + r = run_kubectl("delete", "cluster", cluster_name, "-n", cluster_ns, "--ignore-not-found") + if r.returncode == 0: + result["resources_deleted"].append(f"cluster/{cluster_name}") + + # Wait for cluster resources to be cleaned up + time.sleep(30) + + # Delete BareMetalHosts + bmh_ns = f"{cluster_name}-hosts" + r = run_kubectl("delete", "baremetalhost", "--all", "-n", bmh_ns, "--ignore-not-found") + if r.returncode == 0: + result["resources_deleted"].append(f"baremetalhosts/{bmh_ns}") + + # Delete Carbide instances + for iid in state.get("bmh_instance_ids", []): + r = run_carbide("instance", "delete", iid) + if r.returncode == 0: + result["resources_deleted"].append(f"instance/{iid}") + else: + errors.append(f"instance/{iid}: {r.stderr}") + + # Delete namespaces + for ns in [cluster_ns, bmh_ns]: + run_kubectl("delete", "namespace", ns, "--ignore-not-found") + result["resources_deleted"].append(f"namespace/{ns}") + + # Clean up kubeconfig and state + kubeconfig = state.get("hosted_kubeconfig", "") + if kubeconfig: + Path(kubeconfig).unlink(missing_ok=True) + Path(STATE_FILE).unlink(missing_ok=True) + + if errors: + result["errors"] = errors + result["success"] = len(errors) == 0 + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/hosted/verify_hosted_cluster.py b/isvctl/configs/stubs/openshift/hosted/verify_hosted_cluster.py new file mode 100644 index 00000000..136a2eee --- /dev/null +++ b/isvctl/configs/stubs/openshift/hosted/verify_hosted_cluster.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Verify the hosted cluster is functional. + +Runs basic health checks against the hosted cluster: nodes Ready, +GPU Operator running, nvidia-smi works. + +After this step, the full validation suite can be run by setting +KUBECONFIG to the hosted cluster's kubeconfig. +""" + +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Any + +STATE_FILE = os.environ.get("HOSTED_STATE_FILE", "/tmp/ncp-hosted-state.json") +SCRIPT_DIR = Path(__file__).parent + + +def load_state() -> dict[str, Any]: + try: + return json.loads(Path(STATE_FILE).read_text()) + except (FileNotFoundError, json.JSONDecodeError): + return {} + + +def main() -> int: + state = load_state() + kubeconfig = state.get("hosted_kubeconfig", "") + + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "nodes_ready": 0, + "gpu_operator_running": False, + "kubeconfig_path": kubeconfig, + } + + if not kubeconfig or not Path(kubeconfig).exists(): + result["error"] = "Hosted kubeconfig not found" + print(json.dumps(result, indent=2)) + return 1 + + env = {**os.environ, "KUBECONFIG": kubeconfig} + + # Check nodes + r = subprocess.run( + ["kubectl", "get", "nodes", "--no-headers"], + capture_output=True, text=True, timeout=30, env=env, + ) + if r.returncode == 0: + nodes = [l for l in r.stdout.strip().split("\n") if l.strip()] + ready = sum(1 for l in nodes if "Ready" in l and "NotReady" not in l) + result["nodes_ready"] = ready + result["total_nodes"] = len(nodes) + + # Check GPU Operator + r = subprocess.run( + ["kubectl", "get", "pods", "-n", "nvidia-gpu-operator", + "-l", "app=gpu-operator", "--no-headers"], + capture_output=True, text=True, timeout=30, env=env, + ) + result["gpu_operator_running"] = r.returncode == 0 and "Running" in r.stdout + + # Check GPU nodes + r = subprocess.run( + ["kubectl", "get", "nodes", "-l", "nvidia.com/gpu.present=true", + "--no-headers"], + capture_output=True, text=True, timeout=30, env=env, + ) + gpu_nodes = len([l for l in r.stdout.strip().split("\n") if l.strip()]) if r.returncode == 0 else 0 + result["gpu_nodes"] = gpu_nodes + + result["success"] = result["nodes_ready"] > 0 + result["message"] = ( + f"Hosted cluster: {result['nodes_ready']} nodes Ready, " + f"{gpu_nodes} GPU nodes, " + f"GPU Operator {'running' if result['gpu_operator_running'] else 'not running'}. " + f"Run tests with: KUBECONFIG={kubeconfig} isvctl test run -f ..." + ) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/iam/create_user.py b/isvctl/configs/stubs/openshift/iam/create_user.py new file mode 100644 index 00000000..c3c2185a --- /dev/null +++ b/isvctl/configs/stubs/openshift/iam/create_user.py @@ -0,0 +1,475 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Deploy Keycloak (idempotent) and create a test user. + +Ensures the Red Hat Build of Keycloak (RHBK) operator is installed, +a Keycloak instance is running, the OAuth IdP is configured, and a +test user exists in the validation realm. + +All steps are idempotent — safe to run repeatedly. + +Environment: + KEYCLOAK_NAMESPACE: Namespace for Keycloak (default: ncp-keycloak) + KEYCLOAK_REALM: Realm name (default: ncp-validation) + TEST_USERNAME: Test user name (default: ncp-test-user) + TEST_PASSWORD: Test user password (default: ncp-test-pass) + +Output schema: generic (fields: username, access_key_id) +""" + +import base64 +import json +import os +import ssl +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from typing import Any + + +def run_oc(*args: str, input_data: str | None = None) -> subprocess.CompletedProcess[str]: + cmd = ["oc"] + list(args) + return subprocess.run(cmd, capture_output=True, text=True, input=input_data, timeout=120) + + +def wait_for_deployment(namespace: str, name: str, timeout: int = 300) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + r = run_oc("get", "deployment", name, "-n", namespace, + "-o", "jsonpath={.status.availableReplicas}") + if r.returncode == 0 and r.stdout.strip(): + try: + if int(r.stdout.strip()) > 0: + return True + except ValueError: + pass + time.sleep(10) + return False + + +def wait_for_csv(namespace: str, name_prefix: str, timeout: int = 300) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + r = run_oc("get", "csv", "-n", namespace, + "-o", "jsonpath={range .items[*]}{.metadata.name},{.status.phase}{'\\n'}{end}") + if r.returncode == 0: + for line in r.stdout.strip().split("\n"): + if "," in line: + csv_name, phase = line.rsplit(",", 1) + if csv_name.startswith(name_prefix) and phase == "Succeeded": + return True + time.sleep(10) + return False + + +def keycloak_api(url: str, token: str, method: str = "GET", + data: dict | None = None) -> tuple[int, str]: + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + body = json.dumps(data).encode() if data else None + req = urllib.request.Request(url, data=body, method=method) + req.add_header("Authorization", f"Bearer {token}") + req.add_header("Content-Type", "application/json") + try: + resp = urllib.request.urlopen(req, context=ctx, timeout=30) + return resp.status, resp.read().decode() + except urllib.error.HTTPError as e: + return e.code, e.read().decode() + + +def get_admin_token(keycloak_url: str, username: str, password: str) -> str: + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + token_url = f"{keycloak_url}/realms/master/protocol/openid-connect/token" + body = urllib.parse.urlencode({ + "grant_type": "password", + "client_id": "admin-cli", + "username": username, + "password": password, + }).encode() + req = urllib.request.Request(token_url, data=body, method="POST") + req.add_header("Content-Type", "application/x-www-form-urlencoded") + resp = urllib.request.urlopen(req, context=ctx, timeout=30) + return json.loads(resp.read().decode())["access_token"] + + +def ensure_operator(namespace: str) -> None: + """Ensure RHBK operator is installed (idempotent).""" + # Check if already installed + r = run_oc("get", "csv", "-n", namespace, "--no-headers") + if r.returncode == 0 and "rhbk-operator" in r.stdout: + print("RHBK operator already installed.", file=sys.stderr) + return + + # Create namespace + run_oc("create", "namespace", namespace) + + # OperatorGroup + og_yaml = f""" +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: rhbk-operator-group + namespace: {namespace} +spec: + targetNamespaces: + - {namespace} +""" + run_oc("apply", "-f", "-", input_data=og_yaml) + + # Subscription (Red Hat operator from redhat-operators catalog) + sub_yaml = f""" +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: rhbk-operator + namespace: {namespace} +spec: + channel: stable-v26 + name: rhbk-operator + source: redhat-operators + sourceNamespace: openshift-marketplace + installPlanApproval: Automatic +""" + run_oc("apply", "-f", "-", input_data=sub_yaml) + + print("Waiting for RHBK operator CSV...", file=sys.stderr) + if not wait_for_csv(namespace, "rhbk-operator", timeout=300): + raise RuntimeError("RHBK operator CSV did not reach Succeeded") + + print("Waiting for operator deployment...", file=sys.stderr) + if not wait_for_deployment(namespace, "rhbk-operator", timeout=120): + raise RuntimeError("RHBK operator deployment not available") + + print("RHBK operator installed.", file=sys.stderr) + + +def ensure_keycloak(namespace: str) -> str: + """Ensure Keycloak instance is running (idempotent). Returns route URL.""" + # Check if already running + r = run_oc("get", "keycloak", "ncp-keycloak", "-n", namespace, "--no-headers") + if r.returncode == 0: + print("Keycloak instance already exists.", file=sys.stderr) + else: + kc_yaml = f""" +apiVersion: k8s.keycloak.org/v2alpha1 +kind: Keycloak +metadata: + name: ncp-keycloak + namespace: {namespace} +spec: + instances: 1 + http: + httpEnabled: true + hostname: + strict: false + proxy: + headers: xforwarded + additionalOptions: + - name: http-enabled + value: "true" + - name: hostname-strict + value: "false" + - name: hostname-strict-https + value: "false" +""" + run_oc("apply", "-f", "-", input_data=kc_yaml) + + # Wait for pods + print("Waiting for Keycloak pods...", file=sys.stderr) + deadline = time.time() + 300 + while time.time() < deadline: + r = run_oc("get", "pods", "-n", namespace, "--no-headers") + if r.returncode == 0: + for line in r.stdout.strip().split("\n"): + parts = line.split() + if len(parts) >= 3 and "ncp-keycloak" in parts[0] and "operator" not in parts[0] and parts[2] == "Running": + break + else: + time.sleep(10) + continue + break + time.sleep(10) + + # Ensure route exists (edge termination) + route_yaml = f""" +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: keycloak + namespace: {namespace} +spec: + to: + kind: Service + name: ncp-keycloak-service + port: + targetPort: http + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect +""" + run_oc("apply", "-f", "-", input_data=route_yaml) + + # Get route URL + deadline = time.time() + 60 + while time.time() < deadline: + r = run_oc("get", "routes", "-n", namespace, "-o", "jsonpath={.items[0].spec.host}") + if r.returncode == 0 and r.stdout.strip(): + return f"https://{r.stdout.strip()}" + time.sleep(5) + + raise RuntimeError("Could not get Keycloak route URL") + + +def ensure_realm(keycloak_url: str, token: str, realm_name: str) -> None: + """Ensure realm exists (idempotent).""" + status, _ = keycloak_api(f"{keycloak_url}/admin/realms/{realm_name}", token) + if status == 200: + print(f"Realm '{realm_name}' already exists.", file=sys.stderr) + return + + realm_data = {"realm": realm_name, "enabled": True} + status, body = keycloak_api(f"{keycloak_url}/admin/realms", token, method="POST", data=realm_data) + if status not in (201, 409): + raise RuntimeError(f"Failed to create realm: {status} {body}") + print(f"Realm '{realm_name}' created.", file=sys.stderr) + + +def ensure_client(keycloak_url: str, token: str, realm_name: str) -> str: + """Ensure OpenShift OIDC client exists. Returns client secret.""" + # Check if client exists + status, body = keycloak_api( + f"{keycloak_url}/admin/realms/{realm_name}/clients?clientId=openshift", token) + if status == 200: + clients = json.loads(body) + if clients: + client_uuid = clients[0]["id"] + # Get existing secret + status, body = keycloak_api( + f"{keycloak_url}/admin/realms/{realm_name}/clients/{client_uuid}/client-secret", token) + if status == 200: + return json.loads(body).get("value", "") + + # Get apps domain for redirect URI + r = run_oc("get", "ingress.config.openshift.io", "cluster", "-o", "jsonpath={.spec.domain}") + apps_domain = r.stdout.strip() if r.returncode == 0 else "" + redirect_uri = f"https://oauth-openshift.{apps_domain}/oauth2callback/ncp-keycloak" if apps_domain else "*" + + client_data = { + "clientId": "openshift", + "enabled": True, + "protocol": "openid-connect", + "publicClient": False, + "directAccessGrantsEnabled": True, + "standardFlowEnabled": True, + "redirectUris": [redirect_uri], + } + status, body = keycloak_api( + f"{keycloak_url}/admin/realms/{realm_name}/clients", token, method="POST", data=client_data) + if status not in (201, 409): + raise RuntimeError(f"Failed to create client: {status} {body}") + + # Get client UUID and secret + status, body = keycloak_api( + f"{keycloak_url}/admin/realms/{realm_name}/clients?clientId=openshift", token) + clients = json.loads(body) + client_uuid = clients[0]["id"] + status, body = keycloak_api( + f"{keycloak_url}/admin/realms/{realm_name}/clients/{client_uuid}/client-secret", token) + return json.loads(body).get("value", "") if status == 200 else "" + + +def ensure_oauth(keycloak_url: str, realm_name: str, client_secret: str, + namespace: str) -> None: + """Ensure OpenShift OAuth is configured with Keycloak IdP (idempotent).""" + # Check if already configured + r = run_oc("get", "oauth", "cluster", "-o", "json") + if r.returncode == 0: + oauth = json.loads(r.stdout) + idps = oauth.get("spec", {}).get("identityProviders") or [] + if any(idp.get("name") == "ncp-keycloak" for idp in idps): + print("OAuth IdP already configured.", file=sys.stderr) + return + + # Get ingress CA for TLS verification + r = run_oc("get", "secret", "router-ca", "-n", "openshift-ingress-operator", + "-o", "jsonpath={.data.tls\\.crt}") + ingress_ca = "" + if r.returncode == 0 and r.stdout.strip(): + ingress_ca = base64.b64decode(r.stdout.strip()).decode() + + if ingress_ca: + ca_cm_yaml = "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: ncp-keycloak-ca\n namespace: openshift-config\ndata:\n ca.crt: |\n" + for line in ingress_ca.strip().split("\n"): + ca_cm_yaml += f" {line}\n" + run_oc("apply", "-f", "-", input_data=ca_cm_yaml) + + # Client secret + secret_yaml = f""" +apiVersion: v1 +kind: Secret +metadata: + name: ncp-keycloak-client-secret + namespace: openshift-config +type: Opaque +stringData: + clientSecret: "{client_secret}" +""" + run_oc("apply", "-f", "-", input_data=secret_yaml) + + # Patch OAuth CR + r = run_oc("get", "oauth", "cluster", "-o", "json") + oauth = json.loads(r.stdout) + idps = oauth.get("spec", {}).get("identityProviders") or [] + idps = [idp for idp in idps if idp.get("name") != "ncp-keycloak"] + + issuer = f"{keycloak_url}/realms/{realm_name}" + openid_config: dict[str, Any] = { + "clientID": "openshift", + "clientSecret": {"name": "ncp-keycloak-client-secret"}, + "issuer": issuer, + "claims": { + "preferredUsername": ["preferred_username"], + "name": ["name"], + "email": ["email"], + }, + } + if ingress_ca: + openid_config["ca"] = {"name": "ncp-keycloak-ca"} + + idps.append({ + "name": "ncp-keycloak", + "type": "OpenID", + "mappingMethod": "claim", + "openID": openid_config, + }) + + patch = json.dumps({"spec": {"identityProviders": idps}}) + r = run_oc("patch", "oauth", "cluster", "--type=merge", f"-p={patch}") + if r.returncode != 0: + raise RuntimeError(f"Failed to patch OAuth: {r.stderr}") + + # Wait for auth operator to stabilize + print("Waiting for authentication operator...", file=sys.stderr) + time.sleep(10) + deadline = time.time() + 300 + while time.time() < deadline: + r = run_oc("get", "clusteroperator", "authentication", + "-o", "jsonpath={.status.conditions[?(@.type=='Available')].status}") + if r.returncode == 0 and r.stdout.strip() == "True": + r2 = run_oc("get", "clusteroperator", "authentication", + "-o", "jsonpath={.status.conditions[?(@.type=='Progressing')].status}") + if r2.returncode == 0 and r2.stdout.strip() == "False": + break + time.sleep(15) + + print("OAuth configured with Keycloak IdP.", file=sys.stderr) + + +def create_keycloak_user(keycloak_url: str, token: str, realm_name: str, + username: str, password: str) -> str: + """Create user in Keycloak realm. Returns user ID.""" + # Check if user exists + status, body = keycloak_api( + f"{keycloak_url}/admin/realms/{realm_name}/users?username={username}", token) + if status == 200: + users = json.loads(body) + if users: + user_id = users[0]["id"] + # Update password + keycloak_api( + f"{keycloak_url}/admin/realms/{realm_name}/users/{user_id}/reset-password", + token, method="PUT", + data={"type": "password", "value": password, "temporary": False}) + print(f"User '{username}' already exists, password updated.", file=sys.stderr) + return user_id + + # Create user + user_data = { + "username": username, + "enabled": True, + "credentials": [{"type": "password", "value": password, "temporary": False}], + } + status, body = keycloak_api( + f"{keycloak_url}/admin/realms/{realm_name}/users", token, method="POST", data=user_data) + if status not in (201, 409): + raise RuntimeError(f"Failed to create user: {status} {body}") + + # Get user ID + status, body = keycloak_api( + f"{keycloak_url}/admin/realms/{realm_name}/users?username={username}", token) + users = json.loads(body) + user_id = users[0]["id"] if users else "" + print(f"User '{username}' created.", file=sys.stderr) + return user_id + + +def main() -> int: + namespace = os.environ.get("KEYCLOAK_NAMESPACE", "ncp-keycloak") + realm_name = os.environ.get("KEYCLOAK_REALM", "ncp-validation") + username = os.environ.get("TEST_USERNAME", "ncp-test-user") + password = os.environ.get("TEST_PASSWORD", "ncp-test-pass") + + result: dict[str, Any] = { + "success": False, + "platform": "iam", + "username": username, + } + + try: + # Step 1: Ensure operator + Keycloak instance + ensure_operator(namespace) + keycloak_url = ensure_keycloak(namespace) + result["keycloak_url"] = keycloak_url + + # Step 2: Get admin credentials + for secret_name in ["ncp-keycloak-initial-admin", "keycloak-initial-admin"]: + r = run_oc("get", "secret", secret_name, "-n", namespace, + "-o", "jsonpath={.data.username}") + if r.returncode == 0 and r.stdout.strip(): + admin_user = base64.b64decode(r.stdout.strip()).decode() + r2 = run_oc("get", "secret", secret_name, "-n", namespace, + "-o", "jsonpath={.data.password}") + admin_pass = base64.b64decode(r2.stdout.strip()).decode() + break + else: + raise RuntimeError("Could not get Keycloak admin credentials") + + token = get_admin_token(keycloak_url, admin_user, admin_pass) + + # Step 3: Ensure realm + client + ensure_realm(keycloak_url, token, realm_name) + client_secret = ensure_client(keycloak_url, token, realm_name) + + # Step 4: Configure OAuth (idempotent) + ensure_oauth(keycloak_url, realm_name, client_secret, namespace) + + # Step 5: Create test user + user_id = create_keycloak_user(keycloak_url, token, realm_name, username, password) + result["access_key_id"] = user_id + result["user_id"] = user_id + result["success"] = True + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/iam/teardown.py b/isvctl/configs/stubs/openshift/iam/teardown.py new file mode 100644 index 00000000..ca808fec --- /dev/null +++ b/isvctl/configs/stubs/openshift/iam/teardown.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Clean up test user from Keycloak. Keycloak itself stays deployed. + +Deletes the test user from the Keycloak realm and cleans up +any OpenShift identity/user objects. Does NOT remove Keycloak, +the operator, or the OAuth IdP configuration. + +Environment: + KEYCLOAK_NAMESPACE: Namespace for Keycloak (default: ncp-keycloak) + KEYCLOAK_REALM: Realm name (default: ncp-validation) + TEST_USERNAME: Test user name (default: ncp-test-user) + +Output schema: teardown +""" + +import base64 +import json +import os +import ssl +import subprocess +import sys +import urllib.error +import urllib.parse +import urllib.request +from typing import Any + + +def run_oc(*args: str) -> subprocess.CompletedProcess[str]: + cmd = ["oc"] + list(args) + return subprocess.run(cmd, capture_output=True, text=True, timeout=60) + + +def main() -> int: + namespace = os.environ.get("KEYCLOAK_NAMESPACE", "ncp-keycloak") + realm_name = os.environ.get("KEYCLOAK_REALM", "ncp-validation") + username = os.environ.get("TEST_USERNAME", "ncp-test-user") + + result: dict[str, Any] = { + "success": False, + "platform": "iam", + "resources_deleted": [], + } + + try: + # Get Keycloak route + r = run_oc("get", "routes", "-n", namespace, "-o", "jsonpath={.items[0].spec.host}") + if r.returncode != 0 or not r.stdout.strip(): + result["success"] = True + result["message"] = "Keycloak not found, nothing to clean up" + print(json.dumps(result, indent=2)) + return 0 + keycloak_url = f"https://{r.stdout.strip()}" + + # Get admin token + admin_user = admin_pass = "" + for secret_name in ["ncp-keycloak-initial-admin", "keycloak-initial-admin"]: + r = run_oc("get", "secret", secret_name, "-n", namespace, + "-o", "jsonpath={.data.username}") + if r.returncode == 0 and r.stdout.strip(): + admin_user = base64.b64decode(r.stdout.strip()).decode() + r2 = run_oc("get", "secret", secret_name, "-n", namespace, + "-o", "jsonpath={.data.password}") + admin_pass = base64.b64decode(r2.stdout.strip()).decode() + break + + if not admin_user: + result["success"] = True + result["message"] = "No admin credentials found, skipping user cleanup" + print(json.dumps(result, indent=2)) + return 0 + + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + # Get admin token + token_url = f"{keycloak_url}/realms/master/protocol/openid-connect/token" + body = urllib.parse.urlencode({ + "grant_type": "password", + "client_id": "admin-cli", + "username": admin_user, + "password": admin_pass, + }).encode() + req = urllib.request.Request(token_url, data=body, method="POST") + req.add_header("Content-Type", "application/x-www-form-urlencoded") + resp = urllib.request.urlopen(req, context=ctx, timeout=30) + token = json.loads(resp.read().decode())["access_token"] + + # Find and delete user + req = urllib.request.Request( + f"{keycloak_url}/admin/realms/{realm_name}/users?username={username}", + method="GET") + req.add_header("Authorization", f"Bearer {token}") + resp = urllib.request.urlopen(req, context=ctx, timeout=30) + users = json.loads(resp.read().decode()) + + if users: + user_id = users[0]["id"] + req = urllib.request.Request( + f"{keycloak_url}/admin/realms/{realm_name}/users/{user_id}", + method="DELETE") + req.add_header("Authorization", f"Bearer {token}") + urllib.request.urlopen(req, context=ctx, timeout=30) + result["resources_deleted"].append(f"keycloak-user/{username}") + + except Exception as e: + # Non-fatal — user may already be deleted + print(f"Warning: {e}", file=sys.stderr) + + # Clean up OpenShift identity/user objects + run_oc("delete", "identity", f"ncp-keycloak:{username}", "--ignore-not-found") + result["resources_deleted"].append(f"identity/ncp-keycloak:{username}") + + run_oc("delete", "user", username, "--ignore-not-found") + result["resources_deleted"].append(f"user/{username}") + + result["success"] = True + result["message"] = "Test user cleaned up; Keycloak still deployed" + + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/iam/test_credentials.py b/isvctl/configs/stubs/openshift/iam/test_credentials.py new file mode 100644 index 00000000..2328d2a9 --- /dev/null +++ b/isvctl/configs/stubs/openshift/iam/test_credentials.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Test Keycloak user credentials via oc login and RBAC checks. + +Logs in with the test user, verifies identity, and checks RBAC +grant/deny. Uses a separate kubeconfig to avoid overwriting the +admin session. + +Environment: + KEYCLOAK_NAMESPACE: Namespace for Keycloak (default: ncp-keycloak) + KEYCLOAK_REALM: Realm name (default: ncp-validation) + TEST_USERNAME: Test user name (default: ncp-test-user) + TEST_PASSWORD: Test user password (default: ncp-test-pass) + +Output schema: generic (fields: account_id, tests) +""" + +import json +import os +import subprocess +import sys +from typing import Any + + +def run_oc(*args: str, env: dict | None = None) -> subprocess.CompletedProcess[str]: + cmd = ["oc"] + list(args) + return subprocess.run(cmd, capture_output=True, text=True, timeout=60, + env=env or os.environ) + + +def main() -> int: + username = os.environ.get("TEST_USERNAME", "ncp-test-user") + password = os.environ.get("TEST_PASSWORD", "ncp-test-pass") + + result: dict[str, Any] = { + "success": False, + "platform": "iam", + "account_id": "", + "tests": {}, + } + + # Get API server + r = run_oc("whoami", "--show-server") + if r.returncode != 0: + result["error"] = "Could not get API server URL" + print(json.dumps(result, indent=2)) + return 1 + api_server = r.stdout.strip() + + # Login with test user (separate kubeconfig) + test_kubeconfig = "/tmp/ncp-iam-test-kubeconfig" + test_env = {**os.environ, "KUBECONFIG": test_kubeconfig} + + r = subprocess.run( + ["oc", "login", api_server, + f"--username={username}", f"--password={password}", + "--insecure-skip-tls-verify"], + capture_output=True, text=True, timeout=60, env=test_env, + ) + + if r.returncode == 0: + result["tests"]["identity"] = {"passed": True, "message": f"oc login succeeded as {username}"} + + # Verify identity + r = subprocess.run(["oc", "whoami"], capture_output=True, text=True, timeout=30, env=test_env) + if r.returncode == 0: + identity = r.stdout.strip() + result["account_id"] = identity + result["tests"]["access"] = {"passed": identity == username, "message": f"whoami: {identity}"} + else: + result["tests"]["access"] = {"passed": False, "message": f"whoami failed: {r.stderr}"} + else: + result["tests"]["identity"] = {"passed": False, "message": f"oc login failed: {r.stderr.strip()}"} + result["tests"]["access"] = {"passed": False, "message": "login failed"} + + # RBAC checks (using admin impersonation) + r = run_oc("auth", "can-i", "list", "nodes", f"--as={username}") + denied_nodes = r.stdout.strip() == "no" + + r = run_oc("auth", "can-i", "delete", "namespaces", f"--as={username}") + denied_ns = r.stdout.strip() == "no" + + result["tests"]["rbac_deny"] = { + "passed": denied_nodes and denied_ns, + "message": f"nodes: {'denied' if denied_nodes else 'ALLOWED'}, " + f"namespaces: {'denied' if denied_ns else 'ALLOWED'}", + } + + result["success"] = all(t.get("passed", False) for t in result["tests"].values()) + + # Clean up test kubeconfig + try: + os.unlink(test_kubeconfig) + except FileNotFoundError: + pass + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/kaas/deprovision.py b/isvctl/configs/stubs/openshift/kaas/deprovision.py new file mode 100644 index 00000000..c57df5d5 --- /dev/null +++ b/isvctl/configs/stubs/openshift/kaas/deprovision.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Deprovision OpenShift cluster and Carbide infrastructure. + +Reverses provisioning in order: +1. Delete OpenShift cluster via aicli +2. Delete instances via carbidecli +3. Delete OperatingSystem via carbidecli +4. Delete VPC via carbidecli (only if created by provision.py) + +Environment variables: + TEARDOWN_ENABLED: Must be "true" to actually delete resources + STATE_FILE: Path to state file (default: /tmp/ncp-ocp-provision-state.json) + AI_URL: Assisted Installer URL (for on-prem) + +Output schema: teardown +""" + +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Any + + +STATE_FILE = os.environ.get("STATE_FILE", "/tmp/ncp-ocp-provision-state.json") + + +def run_carbide(*args: str) -> subprocess.CompletedProcess[str]: + """Run a carbidecli command.""" + cmd = ["carbidecli", "-o", "json"] + list(args) + return subprocess.run(cmd, capture_output=True, text=True, timeout=600) + + +def run_aicli(*args: str) -> subprocess.CompletedProcess[str]: + """Run an aicli command.""" + cmd = ["aicli"] + list(args) + ai_url = os.environ.get("AI_URL") + if ai_url: + cmd = ["aicli", "--url", ai_url] + list(args) + return subprocess.run(cmd, capture_output=True, text=True, timeout=300) + + +def load_state() -> dict[str, Any]: + path = Path(STATE_FILE) + return json.loads(path.read_text()) if path.exists() else {} + + +def main() -> int: + teardown_enabled = os.environ.get("TEARDOWN_ENABLED", "false").lower() == "true" + result: dict[str, Any] = { + "success": False, + "platform": "kubernetes", + "resources_deleted": [], + } + + if not teardown_enabled: + result["success"] = True + result["skipped"] = True + result["message"] = "Teardown skipped (TEARDOWN_ENABLED != true)" + print(json.dumps(result, indent=2)) + return 0 + + state = load_state() + if not state: + result["success"] = True + result["message"] = "No provisioning state found" + print(json.dumps(result, indent=2)) + return 0 + + errors = [] + + # 1. Delete OpenShift cluster via aicli + cluster_name = state.get("cluster_name") + if cluster_name and state.get("cluster_installed"): + print(f"Deleting OpenShift cluster '{cluster_name}'...", file=sys.stderr) + r = run_aicli("delete", "cluster", cluster_name) + if r.returncode == 0: + result["resources_deleted"].append("cluster") + else: + errors.append(f"cluster: {r.stderr}") + + # 2. Delete instances (only if we created them) + if state.get("instances_created") and state.get("instance_ids"): + print(f"Deleting {len(state['instance_ids'])} instances...", file=sys.stderr) + for instance_id in state["instance_ids"]: + r = run_carbide("instance", "delete", instance_id) + if r.returncode == 0: + print(f" Deleted instance {instance_id}", file=sys.stderr) + else: + errors.append(f"instance/{instance_id}: {r.stderr}") + result["resources_deleted"].append("instances") + + # 3. Delete OperatingSystem (only if we created it) + if state.get("os_created") and state.get("os_id"): + print(f"Deleting OperatingSystem {state['os_id']}...", file=sys.stderr) + r = run_carbide("operating-system", "delete", state["os_id"]) + if r.returncode == 0: + result["resources_deleted"].append("operating-system") + else: + errors.append(f"operating-system: {r.stderr}") + + # 4. Delete VPC (only if we created it) + if state.get("vpc_created") and state.get("vpc_id"): + print(f"Deleting VPC {state['vpc_id']}...", file=sys.stderr) + r = run_carbide("vpc", "delete", state["vpc_id"]) + if r.returncode == 0: + result["resources_deleted"].append("vpc") + else: + errors.append(f"vpc: {r.stderr}") + + # Clean up state file + Path(STATE_FILE).unlink(missing_ok=True) + + if errors: + result["errors"] = errors + result["success"] = False + else: + result["success"] = True + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/kaas/provision.py b/isvctl/configs/stubs/openshift/kaas/provision.py new file mode 100644 index 00000000..68908836 --- /dev/null +++ b/isvctl/configs/stubs/openshift/kaas/provision.py @@ -0,0 +1,426 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Provision OpenShift cluster on Carbide bare metal. + +Orchestrates Assisted Installer (aicli) and Carbide (carbidecli) to: +1. Create a cluster in Assisted Installer +2. Collect the iPXE discovery config +3. Create an OperatingSystem in Carbide with the iPXE config +4. Batch-create bare-metal instances with the OS + InstanceType +5. Collect instance UUIDs + MAC addresses +6. Monitor hosts calling home in AI, match to instances, approve +7. Monitor OpenShift installation and collect kubeconfig +8. Install GPU Operator via OperatorHub + +Supports pre-existing Carbide resources (VPC, subnet, etc.) via env vars. + +Environment variables: + CARBIDE_TOKEN / CARBIDE_API_KEY: Carbide authentication + CARBIDE_ORG: Carbide organization + CARBIDE_SITE_ID: Site UUID for instance provisioning + CARBIDE_INSTANCE_TYPE: Instance type UUID + CARBIDE_VPC_ID: Pre-existing VPC UUID (optional, creates if unset) + CARBIDE_SUBNET_ID: Pre-existing subnet UUID (optional) + CARBIDE_SSH_KEY_GROUP: SSH key group UUID (optional) + CARBIDE_INSTANCE_COUNT: Number of instances (default: 3) + OCP_PULL_SECRET: OpenShift pull secret (JSON string or file path) + OCP_VERSION: OpenShift version (default: 4.18) + OCP_BASE_DOMAIN: Base DNS domain (default: example.com) + AI_OFFLINETOKEN: Assisted Installer offline token (SaaS) + AI_URL: Assisted Installer URL (on-prem, overrides SaaS) + STATE_FILE: State file path (default: /tmp/ncp-ocp-provision-state.json) + +Output schema: cluster +""" + +import json +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "stubs")) +try: + from carbide.common.carbide import run_carbide, load_state, save_state +except ImportError: + # Fallback if import path doesn't resolve + def run_carbide(*args: str, timeout: int = 600) -> dict[str, Any]: + cmd = ["carbidecli", "-o", "json"] + list(args) + result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + if result.returncode != 0: + raise RuntimeError(f"carbidecli {' '.join(args)} failed: {result.stderr}") + return json.loads(result.stdout) + + def load_state(state_file: str | None = None) -> dict[str, Any]: + path = Path(state_file or os.environ.get("STATE_FILE", "/tmp/ncp-ocp-provision-state.json")) + return json.loads(path.read_text()) if path.exists() else {} + + def save_state(state: dict[str, Any], state_file: str | None = None) -> None: + path = Path(state_file or os.environ.get("STATE_FILE", "/tmp/ncp-ocp-provision-state.json")) + path.write_text(json.dumps(state, indent=2)) + + +STATE_FILE = os.environ.get("STATE_FILE", "/tmp/ncp-ocp-provision-state.json") + + +def run_aicli(*args: str, timeout: int = 300) -> subprocess.CompletedProcess[str]: + """Run an aicli command.""" + cmd = ["aicli"] + list(args) + ai_url = os.environ.get("AI_URL") + if ai_url: + cmd = ["aicli", "--url", ai_url] + list(args) + return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + + +def step_create_ai_cluster(state: dict[str, Any]) -> dict[str, Any]: + """Step 1: Create cluster in Assisted Installer.""" + if state.get("ai_cluster_created"): + print("AI cluster already created, skipping...", file=sys.stderr) + return state + + cluster_name = os.environ.get("CARBIDE_VPC_NAME", "ncp-ocp-validation") + ocp_version = os.environ.get("OCP_VERSION", "4.18") + base_domain = os.environ.get("OCP_BASE_DOMAIN", "example.com") + pull_secret = os.environ.get("OCP_PULL_SECRET", "") + + if not pull_secret: + raise RuntimeError("OCP_PULL_SECRET is required") + + # Resolve pull secret to file path (aicli expects a file) + if pull_secret and not os.path.isfile(pull_secret): + pull_secret_path = Path(f"/tmp/{cluster_name}-pull-secret.json") + pull_secret_path.write_text(pull_secret) + pull_secret = str(pull_secret_path) + + print(f"Creating OpenShift {ocp_version} cluster '{cluster_name}'...", file=sys.stderr) + create_args = [ + "create", "cluster", cluster_name, + "-P", f"openshift_version={ocp_version}", + "-P", f"base_dns_domain={base_domain}", + ] + if pull_secret: + create_args += ["-P", f"pull_secret={pull_secret}"] + + r = run_aicli(*create_args) + if r.returncode != 0: + raise RuntimeError(f"aicli create cluster failed: {r.stderr}") + + state["cluster_name"] = cluster_name + state["ocp_version"] = ocp_version + state["ai_cluster_created"] = True + save_state(state, STATE_FILE) + return state + + +def step_collect_ipxe(state: dict[str, Any]) -> dict[str, Any]: + """Step 2: Collect iPXE discovery config from AI.""" + if state.get("ipxe_config"): + print("iPXE config already collected, skipping...", file=sys.stderr) + return state + + cluster_name = state["cluster_name"] + print("Collecting iPXE discovery config...", file=sys.stderr) + + r = run_aicli("info", "iso", cluster_name) + if r.returncode != 0: + raise RuntimeError(f"Failed to get discovery ISO info: {r.stderr}") + + # The iPXE config is generated from the discovery ISO URL + iso_url = r.stdout.strip() + ipxe_script = f"""#!ipxe +chain {iso_url} +""" + state["ipxe_config"] = ipxe_script + state["discovery_iso_url"] = iso_url + save_state(state, STATE_FILE) + print(f" iPXE config collected (ISO URL: {iso_url[:80]}...)", file=sys.stderr) + return state + + +def step_create_os(state: dict[str, Any]) -> dict[str, Any]: + """Step 3: Create OperatingSystem in Carbide with iPXE config.""" + if state.get("os_id"): + print("OperatingSystem already created, skipping...", file=sys.stderr) + return state + + os_name = f"{state['cluster_name']}-discovery" + print(f"Creating OperatingSystem '{os_name}'...", file=sys.stderr) + + result = run_carbide( + "operating-system", "create", + "--name", os_name, + "--type", "iPXE", + "--ipxe-script", state["ipxe_config"], + ) + state["os_id"] = result.get("id") + state["os_name"] = os_name + state["os_created"] = True + save_state(state, STATE_FILE) + print(f" OperatingSystem created: {state['os_id']}", file=sys.stderr) + return state + + +def step_create_instances(state: dict[str, Any]) -> dict[str, Any]: + """Step 4: Batch-create bare-metal instances.""" + if state.get("instance_ids"): + print("Instances already created, skipping...", file=sys.stderr) + return state + + site_id = os.environ.get("CARBIDE_SITE_ID", "") + instance_type = os.environ.get("CARBIDE_INSTANCE_TYPE", "") + instance_count = int(os.environ.get("CARBIDE_INSTANCE_COUNT", "3")) + vpc_id = os.environ.get("CARBIDE_VPC_ID", state.get("vpc_id", "")) + ssh_key_group = os.environ.get("CARBIDE_SSH_KEY_GROUP", "") + + if not site_id: + raise RuntimeError("CARBIDE_SITE_ID is required") + if not instance_type: + raise RuntimeError("CARBIDE_INSTANCE_TYPE is required") + + # Create VPC if not pre-existing + if not vpc_id: + print("Creating VPC...", file=sys.stderr) + result = run_carbide("vpc", "create", + "--name", f"{state['cluster_name']}-vpc", + "--site-id", site_id) + vpc_id = result.get("id") + state["vpc_id"] = vpc_id + state["vpc_created"] = True + save_state(state, STATE_FILE) + + print(f"Creating {instance_count} instances...", file=sys.stderr) + batch_args = [ + "instance", "batch-create", + "--name-prefix", f"{state['cluster_name']}-node", + "--count", str(instance_count), + "--vpc-id", vpc_id, + "--instance-type", instance_type, + "--operating-system-id", state["os_id"], + ] + if ssh_key_group: + batch_args += ["--ssh-key-group-ids", ssh_key_group] + + result = run_carbide(*batch_args) + instances = result if isinstance(result, list) else result.get("instances", [result]) + instance_ids = [i["id"] for i in instances] + + state["instance_ids"] = instance_ids + state["instances_created"] = True + save_state(state, STATE_FILE) + print(f" Instances created: {instance_ids}", file=sys.stderr) + return state + + +def step_collect_instance_info(state: dict[str, Any]) -> dict[str, Any]: + """Step 5: Collect instance UUIDs + MAC addresses.""" + if state.get("instance_macs"): + return state + + print("Collecting instance info (UUIDs + MACs)...", file=sys.stderr) + macs = {} + for instance_id in state["instance_ids"]: + info = run_carbide("instance", "get", instance_id) + mac = info.get("macAddress", info.get("mac_address", "")) + macs[instance_id] = mac + print(f" {instance_id}: MAC={mac}", file=sys.stderr) + + state["instance_macs"] = macs + save_state(state, STATE_FILE) + return state + + +def step_monitor_hosts(state: dict[str, Any]) -> dict[str, Any]: + """Step 6: Monitor hosts in AI, match to instances, approve.""" + if state.get("hosts_approved"): + return state + + cluster_name = state["cluster_name"] + instance_count = len(state["instance_ids"]) + + print(f"Waiting for {instance_count} hosts to register in AI...", file=sys.stderr) + r = run_aicli("wait", "hosts", cluster_name, "-n", str(instance_count), + timeout=3600) + if r.returncode != 0: + raise RuntimeError(f"Hosts did not register: {r.stderr}") + print(" All hosts registered", file=sys.stderr) + + state["hosts_approved"] = True + save_state(state, STATE_FILE) + return state + + +def step_install_openshift(state: dict[str, Any]) -> dict[str, Any]: + """Step 7: Start and monitor OpenShift installation.""" + if state.get("cluster_installed"): + return state + + cluster_name = state["cluster_name"] + + print("Starting OpenShift installation...", file=sys.stderr) + r = run_aicli("start", cluster_name) + if r.returncode != 0: + raise RuntimeError(f"aicli start failed: {r.stderr}") + + print("Waiting for OpenShift installation to complete...", file=sys.stderr) + r = run_aicli("wait", cluster_name, timeout=5400) # 90 minutes + if r.returncode != 0: + raise RuntimeError(f"OpenShift installation did not complete: {r.stderr}") + print(" OpenShift installation complete", file=sys.stderr) + + # Download kubeconfig + kubeconfig_path = os.environ.get("KUBECONFIG", f"/tmp/{cluster_name}-kubeconfig") + r = run_aicli("download", "kubeconfig", cluster_name) + if r.returncode == 0: + local_kubeconfig = Path(f"kubeconfig.{cluster_name}") + if local_kubeconfig.exists(): + local_kubeconfig.rename(kubeconfig_path) + + state["cluster_installed"] = True + state["kubeconfig"] = kubeconfig_path + save_state(state, STATE_FILE) + return state + + +def step_day2_operators(state: dict[str, Any]) -> dict[str, Any]: + """Step 8: Install day-2 operators (GPU Operator, etc.).""" + if state.get("operators_installed"): + return state + + kubeconfig = state.get("kubeconfig", os.environ.get("KUBECONFIG", "")) + oc_cmd = ["oc", "--kubeconfig", kubeconfig] if kubeconfig else ["oc"] + + # Install GPU Operator + print("Installing GPU Operator via OperatorHub...", file=sys.stderr) + subprocess.run(oc_cmd + ["create", "namespace", "nvidia-gpu-operator"], + capture_output=True, text=True) + + og_yaml = """ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: nvidia-gpu-operator + namespace: nvidia-gpu-operator +spec: + targetNamespaces: + - nvidia-gpu-operator +""" + subprocess.run(oc_cmd + ["apply", "-f", "-"], + input=og_yaml, capture_output=True, text=True) + + subscription_yaml = """ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: gpu-operator-certified + namespace: nvidia-gpu-operator +spec: + channel: "v24.9" + installPlanApproval: Automatic + name: gpu-operator-certified + source: certified-operators + sourceNamespace: openshift-marketplace +""" + result = subprocess.run(oc_cmd + ["apply", "-f", "-"], + input=subscription_yaml, capture_output=True, text=True) + if result.returncode != 0: + print(f"Warning: GPU Operator subscription failed: {result.stderr}", file=sys.stderr) + + # Wait for GPU Operator pods + max_wait = 600 + elapsed = 0 + while elapsed < max_wait: + result = subprocess.run( + oc_cmd + ["get", "pods", "-n", "nvidia-gpu-operator", + "-l", "app=gpu-operator", "--no-headers"], + capture_output=True, text=True, + ) + if "Running" in result.stdout: + print(" GPU Operator is running", file=sys.stderr) + break + time.sleep(30) + elapsed += 30 + + state["operators_installed"] = True + state["gpu_operator_namespace"] = "nvidia-gpu-operator" + save_state(state, STATE_FILE) + return state + + +def collect_inventory(state: dict[str, Any]) -> dict[str, Any]: + """Collect cluster inventory for validation output.""" + kubeconfig = state.get("kubeconfig", os.environ.get("KUBECONFIG", "")) + + # Query cluster info via oc/kubectl + env = {**os.environ, "KUBECONFIG": kubeconfig} if kubeconfig else dict(os.environ) + + setup_script = Path(__file__).parent / "setup.sh" + if setup_script.exists(): + result = subprocess.run( + ["bash", str(setup_script)], + capture_output=True, text=True, env=env, + ) + if result.returncode == 0: + return json.loads(result.stdout) + + # Fallback: minimal inventory + return { + "success": True, + "platform": "kubernetes", + "cluster_name": state.get("cluster_name", ""), + "node_count": len(state.get("instance_ids", [])), + "kubernetes": { + "node_count": len(state.get("instance_ids", [])), + "gpu_operator_namespace": state.get("gpu_operator_namespace", "nvidia-gpu-operator"), + }, + } + + +def main() -> int: + # Validate prerequisites + for tool in ["carbidecli", "aicli"]: + if subprocess.run([tool, "--help"], capture_output=True).returncode != 0: + print(json.dumps({ + "success": False, + "platform": "kubernetes", + "error": f"{tool} not found", + }, indent=2)) + return 1 + + try: + state = load_state(STATE_FILE) + + state = step_create_ai_cluster(state) + state = step_collect_ipxe(state) + state = step_create_os(state) + state = step_create_instances(state) + state = step_collect_instance_info(state) + state = step_monitor_hosts(state) + state = step_install_openshift(state) + state = step_day2_operators(state) + + inventory = collect_inventory(state) + print(json.dumps(inventory, indent=2)) + return 0 + + except Exception as e: + print(json.dumps({ + "success": False, + "platform": "kubernetes", + "error": str(e), + }, indent=2)) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/kaas/setup.sh b/isvctl/configs/stubs/openshift/kaas/setup.sh new file mode 100755 index 00000000..25ea213a --- /dev/null +++ b/isvctl/configs/stubs/openshift/kaas/setup.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# OpenShift KaaS Setup - Query existing cluster inventory +# +# Sources the shared K8s inventory script for generic kubectl queries, +# then adds OpenShift-specific info (cluster version, infrastructure name). +# +# Requirements: +# - kubectl and oc CLI configured and authenticated +# - jq +# +# Output: JSON inventory conforming to isvctl cluster schema + +set -eo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# OpenShift defaults (before sourcing shared script) +GPU_OPERATOR_NS="${GPU_OPERATOR_NS:-nvidia-gpu-operator}" + +# Collect generic K8s inventory (sets NODE_COUNT, NODES, GPU_*, DRIVER_*, etc.) +# shellcheck source=../../common/k8s-inventory.sh +source "${SCRIPT_DIR}/../../common/k8s-inventory.sh" + +# ----------------------------------------------------------------------------- +# OpenShift-specific info (oc) +# ----------------------------------------------------------------------------- + +if command -v oc &> /dev/null; then + CLUSTER_NAME=$(oc get infrastructure cluster -o jsonpath='{.status.infrastructureName}' 2>/dev/null || echo "openshift-cluster") + OCP_VERSION=$(oc get clusterversion version -o jsonpath='{.status.desired.version}' 2>/dev/null || echo "") + echo "OpenShift: ${OCP_VERSION} (${CLUSTER_NAME})" >&2 +else + CLUSTER_NAME="openshift-cluster" + OCP_VERSION="" +fi + +# OpenShift doesn't use RuntimeClass for GPU pods +RUNTIME_CLASS="${RUNTIME_CLASS:-}" + +# ----------------------------------------------------------------------------- +# Output JSON Inventory +# ----------------------------------------------------------------------------- + +cat << EOF +{ + "success": true, + "platform": "kubernetes", + "cluster_name": "${CLUSTER_NAME}", + "node_count": ${NODE_COUNT}, + "endpoint": "${API_SERVER}", + "gpu_count": ${TOTAL_GPUS}, + "gpu_per_node": ${GPU_PER_NODE}, + "driver_version": "${DRIVER_VERSION}", + "kubeconfig_path": "${KUBECONFIG_PATH}", + "kubernetes": { + "driver_version": "${DRIVER_VERSION}", + "node_count": ${NODE_COUNT}, + "nodes": ${NODES}, + "gpu_node_count": ${GPU_NODE_COUNT}, + "gpu_per_node": ${GPU_PER_NODE}, + "total_gpus": ${TOTAL_GPUS}, + "control_plane_address": "${API_SERVER}", + "kubeconfig_path": "${KUBECONFIG_PATH}", + "gpu_operator_namespace": "${GPU_OPERATOR_NS}", + "runtime_class": "${RUNTIME_CLASS}", + "gpu_resource_name": "nvidia.com/gpu" + }, + "openshift": { + "version": "${OCP_VERSION}", + "cluster_name": "${CLUSTER_NAME}", + "gpu_product": "${GPU_PRODUCT}" + } +} +EOF diff --git a/isvctl/configs/stubs/openshift/kaas/teardown.sh b/isvctl/configs/stubs/openshift/kaas/teardown.sh new file mode 100755 index 00000000..ccad25e3 --- /dev/null +++ b/isvctl/configs/stubs/openshift/kaas/teardown.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# OpenShift KaaS Teardown - Clean up test namespace +# +# Deletes the test namespace if it was created during validation. +# Does not deprovision the cluster itself. + +set -eo pipefail + +NAMESPACE="${K8S_NAMESPACE:-ncp-validation}" + +echo "Cleaning up namespace ${NAMESPACE}..." >&2 + +kubectl delete namespace "${NAMESPACE}" --ignore-not-found 2>/dev/null || true + +cat << EOF +{ + "success": true, + "platform": "kubernetes", + "resources_deleted": ["namespace/${NAMESPACE}"], + "message": "Test namespace cleaned up" +} +EOF diff --git a/isvctl/configs/stubs/openshift/machineset/create_machineset.py b/isvctl/configs/stubs/openshift/machineset/create_machineset.py new file mode 100644 index 00000000..92515fa9 --- /dev/null +++ b/isvctl/configs/stubs/openshift/machineset/create_machineset.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Create a GPU MachineSet and wait for nodes to join. + +Creates a MachineSet targeting Carbide GPU instance type with the +specified initial replicas, then waits for all machines to be +provisioned and their corresponding nodes to reach Ready state. + +Environment: + CARBIDE_INSTANCE_TYPE: Instance type for GPU machines + CARBIDE_SITE_ID: Site UUID + MACHINESET_MIN_REPLICAS: Initial replicas (default: 2) + MACHINESET_MAX_REPLICAS: Max replicas for autoscaler (default: 6) + MACHINESET_NAME: MachineSet name (default: ncp-gpu-workers) + +Output: {"success": true, "replicas_ready": 2, "nodes_joined": 2} +""" + +import json +import os +import subprocess +import sys +import time +from typing import Any + + +NAMESPACE = "openshift-machine-api" + + +def run_oc(*args: str, input_data: str | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run(["oc"] + list(args), capture_output=True, text=True, + input=input_data, timeout=120) + + +def run_kubectl(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, timeout=120) + + +def main() -> int: + instance_type = os.environ.get("CARBIDE_INSTANCE_TYPE", "") + site_id = os.environ.get("CARBIDE_SITE_ID", "") + min_replicas = int(os.environ.get("MACHINESET_MIN_REPLICAS", "2")) + max_replicas = int(os.environ.get("MACHINESET_MAX_REPLICAS", "6")) + ms_name = os.environ.get("MACHINESET_NAME", "ncp-gpu-workers") + + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "machineset_name": ms_name, + "replicas_ready": 0, + "nodes_joined": 0, + } + + if not instance_type: + result["error"] = "CARBIDE_INSTANCE_TYPE is required" + print(json.dumps(result, indent=2)) + return 1 + + try: + # Get cluster infrastructure name for MachineSet naming + r = run_oc("get", "infrastructure", "cluster", + "-o", "jsonpath={.status.infrastructureName}") + infra_name = r.stdout.strip() if r.returncode == 0 else "cluster" + + # Check if MachineSet already exists + r = run_oc("get", "machineset", ms_name, "-n", NAMESPACE, "--no-headers") + if r.returncode == 0 and r.stdout.strip(): + print(f"MachineSet {ms_name} already exists.", file=sys.stderr) + else: + # Create MachineSet + # The providerSpec depends on the Carbide Machine API provider + ms_yaml = f""" +apiVersion: machine.openshift.io/v1beta1 +kind: MachineSet +metadata: + name: {ms_name} + namespace: {NAMESPACE} + labels: + machine.openshift.io/cluster-api-cluster: {infra_name} +spec: + replicas: {min_replicas} + selector: + matchLabels: + machine.openshift.io/cluster-api-cluster: {infra_name} + machine.openshift.io/cluster-api-machineset: {ms_name} + template: + metadata: + labels: + machine.openshift.io/cluster-api-cluster: {infra_name} + machine.openshift.io/cluster-api-machineset: {ms_name} + machine.openshift.io/cluster-api-machine-role: worker + machine.openshift.io/cluster-api-machine-type: worker + spec: + providerSpec: + value: + apiVersion: carbide.nvidia.com/v1alpha1 + kind: CarbideMachineProviderConfig + instanceType: "{instance_type}" + siteId: "{site_id}" + metadata: + labels: + node-role.kubernetes.io/worker: "" + nvidia.com/gpu.present: "true" +""" + r = run_oc("apply", "-f", "-", input_data=ms_yaml) + if r.returncode != 0: + result["error"] = f"Failed to create MachineSet: {r.stderr}" + print(json.dumps(result, indent=2)) + return 1 + + print(f"MachineSet {ms_name} created with {min_replicas} replicas.", file=sys.stderr) + + # Create ClusterAutoscaler and MachineAutoscaler if max > min + if max_replicas > min_replicas: + ma_yaml = f""" +apiVersion: autoscaling.openshift.io/v1beta1 +kind: MachineAutoscaler +metadata: + name: {ms_name}-autoscaler + namespace: {NAMESPACE} +spec: + minReplicas: {min_replicas} + maxReplicas: {max_replicas} + scaleTargetRef: + apiVersion: machine.openshift.io/v1beta1 + kind: MachineSet + name: {ms_name} +""" + run_oc("apply", "-f", "-", input_data=ma_yaml) + print(f"MachineAutoscaler created (min={min_replicas}, max={max_replicas}).", file=sys.stderr) + + # Wait for machines to be provisioned + print("Waiting for machines to provision...", file=sys.stderr) + deadline = time.time() + 1500 # 25 minutes for BM provisioning + while time.time() < deadline: + r = run_oc("get", "machineset", ms_name, "-n", NAMESPACE, + "-o", "jsonpath={.status.readyReplicas}") + ready = int(r.stdout.strip()) if r.returncode == 0 and r.stdout.strip() else 0 + if ready >= min_replicas: + result["replicas_ready"] = ready + break + print(f" Ready: {ready}/{min_replicas}", file=sys.stderr) + time.sleep(30) + + # Count GPU nodes that joined + r = run_kubectl("get", "nodes", "-l", + f"machine.openshift.io/cluster-api-machineset={ms_name}", + "--no-headers") + nodes = [l for l in r.stdout.strip().split("\n") if l.strip()] if r.returncode == 0 else [] + result["nodes_joined"] = len(nodes) + result["node_names"] = [n.split()[0] for n in nodes] + + result["success"] = result["replicas_ready"] >= min_replicas + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/machineset/teardown.py b/isvctl/configs/stubs/openshift/machineset/teardown.py new file mode 100644 index 00000000..8d105d06 --- /dev/null +++ b/isvctl/configs/stubs/openshift/machineset/teardown.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Delete MachineSet, MachineAutoscaler, and test namespace. + +Environment: + MACHINESET_NAME: MachineSet name (default: ncp-gpu-workers) + TEARDOWN_ENABLED: Must be "true" to delete MachineSet (default: false) + +Output: {"success": true, "resources_deleted": [...]} +""" + +import json +import os +import subprocess +import sys +import time +from typing import Any + +MS_NAMESPACE = "openshift-machine-api" +TEST_NAMESPACE = os.environ.get("K8S_NAMESPACE", "ncp-machineset-validation") + + +def run_oc(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["oc"] + list(args), capture_output=True, text=True, timeout=120) + + +def run_kubectl(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, timeout=120) + + +def main() -> int: + ms_name = os.environ.get("MACHINESET_NAME", "ncp-gpu-workers") + teardown_enabled = os.environ.get("TEARDOWN_ENABLED", "false").lower() == "true" + + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "resources_deleted": [], + } + + # Always clean up test namespace + run_kubectl("delete", "namespace", TEST_NAMESPACE, "--ignore-not-found") + result["resources_deleted"].append(f"namespace/{TEST_NAMESPACE}") + + if not teardown_enabled: + result["success"] = True + result["message"] = "MachineSet preserved (TEARDOWN_ENABLED != true)" + print(json.dumps(result, indent=2)) + return 0 + + # Delete MachineAutoscaler + run_oc("delete", "machineautoscaler", f"{ms_name}-autoscaler", + "-n", MS_NAMESPACE, "--ignore-not-found") + result["resources_deleted"].append(f"machineautoscaler/{ms_name}-autoscaler") + + # Scale to 0 first (graceful) + run_oc("scale", "machineset", ms_name, "-n", MS_NAMESPACE, "--replicas=0") + print("Waiting for machines to deprovision...", file=sys.stderr) + + deadline = time.time() + 600 + while time.time() < deadline: + r = run_oc("get", "machines", "-n", MS_NAMESPACE, "-l", + f"machine.openshift.io/cluster-api-machineset={ms_name}", + "--no-headers") + if not r.stdout.strip(): + break + time.sleep(15) + + # Delete MachineSet + run_oc("delete", "machineset", ms_name, "-n", MS_NAMESPACE, "--ignore-not-found") + result["resources_deleted"].append(f"machineset/{ms_name}") + + result["success"] = True + + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/machineset/test_gpu_workload.py b/isvctl/configs/stubs/openshift/machineset/test_gpu_workload.py new file mode 100644 index 00000000..01393ee6 --- /dev/null +++ b/isvctl/configs/stubs/openshift/machineset/test_gpu_workload.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Run a single-node GPU workload on the MachineSet nodes. + +Schedules a GPU pod on one of the MachineSet-provisioned nodes to +verify GPU scheduling works on dynamically provisioned machines. + +Output: {"success": true, "gpu_detected": true, "node": ""} +""" + +import json +import os +import subprocess +import sys +import time +from typing import Any + +NAMESPACE = os.environ.get("K8S_NAMESPACE", "ncp-machineset-validation") + + +def run_kubectl(*args: str, input_data: str | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, + input=input_data, timeout=120) + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "gpu_detected": False, + } + + try: + run_kubectl("create", "namespace", NAMESPACE) + + pod_yaml = f""" +apiVersion: v1 +kind: Pod +metadata: + name: ncp-gpu-test + namespace: {NAMESPACE} +spec: + restartPolicy: Never + containers: + - name: gpu-test + image: nvcr.io/nvidia/cuda:12.8.0-base-ubi9 + command: ["nvidia-smi"] + resources: + limits: + nvidia.com/gpu: "1" +""" + run_kubectl("delete", "pod", "ncp-gpu-test", "-n", NAMESPACE, "--ignore-not-found") + run_kubectl("apply", "-f", "-", input_data=pod_yaml) + + # Wait for completion + print("Waiting for GPU test pod...", file=sys.stderr) + deadline = time.time() + 300 + while time.time() < deadline: + r = run_kubectl("get", "pod", "ncp-gpu-test", "-n", NAMESPACE, + "-o", "jsonpath={.status.phase}") + phase = r.stdout.strip() if r.returncode == 0 else "" + if phase == "Succeeded": + result["gpu_detected"] = True + break + if phase == "Failed": + result["error"] = "GPU test pod failed" + break + time.sleep(10) + + # Get which node it ran on + r = run_kubectl("get", "pod", "ncp-gpu-test", "-n", NAMESPACE, + "-o", "jsonpath={.spec.nodeName}") + result["node"] = r.stdout.strip() if r.returncode == 0 else "" + + # Get nvidia-smi output + r = run_kubectl("logs", "ncp-gpu-test", "-n", NAMESPACE) + if r.returncode == 0: + result["nvidia_smi_output"] = r.stdout[:500] + + run_kubectl("delete", "pod", "ncp-gpu-test", "-n", NAMESPACE, "--ignore-not-found") + + result["success"] = result["gpu_detected"] + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/machineset/test_scale_down.py b/isvctl/configs/stubs/openshift/machineset/test_scale_down.py new file mode 100644 index 00000000..2b889134 --- /dev/null +++ b/isvctl/configs/stubs/openshift/machineset/test_scale_down.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Test MachineSet scale-down by reducing replicas. + +Scales the MachineSet back to minimum replicas and verifies that +excess machines are deprovisioned (proves Carbide delete works). + +Environment: + MACHINESET_NAME: MachineSet name (default: ncp-gpu-workers) + MACHINESET_MIN_REPLICAS: Target replicas (default: 2) + +Output: {"success": true, "scaled_to": N, "nodes_after": N} +""" + +import json +import os +import subprocess +import sys +import time +from typing import Any + +NAMESPACE = "openshift-machine-api" + + +def run_oc(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["oc"] + list(args), capture_output=True, text=True, timeout=120) + + +def run_kubectl(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, timeout=120) + + +def main() -> int: + ms_name = os.environ.get("MACHINESET_NAME", "ncp-gpu-workers") + target = int(os.environ.get("MACHINESET_MIN_REPLICAS", "2")) + + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + } + + try: + # Get current replicas + r = run_oc("get", "machineset", ms_name, "-n", NAMESPACE, + "-o", "jsonpath={.spec.replicas}") + current = int(r.stdout.strip()) if r.returncode == 0 and r.stdout.strip() else 0 + result["replicas_before"] = current + + if current <= target: + result["success"] = True + result["scaled_to"] = current + result["nodes_after"] = current + result["message"] = "Already at or below target replicas" + print(json.dumps(result, indent=2)) + return 0 + + # Scale down + print(f"Scaling {ms_name} from {current} to {target}...", file=sys.stderr) + r = run_oc("scale", "machineset", ms_name, "-n", NAMESPACE, + f"--replicas={target}") + if r.returncode != 0: + result["error"] = f"Scale failed: {r.stderr}" + print(json.dumps(result, indent=2)) + return 1 + + # Wait for machines to be removed + print("Waiting for machines to deprovision...", file=sys.stderr) + deadline = time.time() + 900 + while time.time() < deadline: + r = run_oc("get", "machineset", ms_name, "-n", NAMESPACE, + "-o", "jsonpath={.status.replicas}") + actual = int(r.stdout.strip()) if r.returncode == 0 and r.stdout.strip() else current + if actual <= target: + break + print(f" Replicas: {actual}/{target}", file=sys.stderr) + time.sleep(30) + + # Count remaining nodes + r = run_kubectl("get", "nodes", "-l", + f"machine.openshift.io/cluster-api-machineset={ms_name}", + "--no-headers") + nodes = [l for l in r.stdout.strip().split("\n") if l.strip()] if r.returncode == 0 else [] + + result["scaled_to"] = target + result["nodes_after"] = len(nodes) + result["machines_deprovisioned"] = current - len(nodes) + result["success"] = len(nodes) <= target + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/machineset/test_scale_up.py b/isvctl/configs/stubs/openshift/machineset/test_scale_up.py new file mode 100644 index 00000000..9e4b8cde --- /dev/null +++ b/isvctl/configs/stubs/openshift/machineset/test_scale_up.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Test MachineSet scale-up by increasing replicas. + +Scales the MachineSet up by adding replicas and waits for the new +machines to provision and join as Ready nodes. + +Environment: + MACHINESET_NAME: MachineSet name (default: ncp-gpu-workers) + MACHINESET_SCALE_UP_TO: Target replicas (default: current + 1) + +Output: {"success": true, "scaled_to": N, "nodes_after": N} +""" + +import json +import os +import subprocess +import sys +import time +from typing import Any + +NAMESPACE = "openshift-machine-api" + + +def run_oc(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["oc"] + list(args), capture_output=True, text=True, timeout=120) + + +def run_kubectl(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, timeout=120) + + +def main() -> int: + ms_name = os.environ.get("MACHINESET_NAME", "ncp-gpu-workers") + + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + } + + try: + # Get current replicas + r = run_oc("get", "machineset", ms_name, "-n", NAMESPACE, + "-o", "jsonpath={.spec.replicas}") + current = int(r.stdout.strip()) if r.returncode == 0 and r.stdout.strip() else 0 + result["replicas_before"] = current + + target = int(os.environ.get("MACHINESET_SCALE_UP_TO", str(current + 1))) + if target <= current: + result["success"] = True + result["scaled_to"] = current + result["nodes_after"] = current + result["message"] = "Already at or above target replicas" + print(json.dumps(result, indent=2)) + return 0 + + # Scale up + print(f"Scaling {ms_name} from {current} to {target}...", file=sys.stderr) + r = run_oc("scale", "machineset", ms_name, "-n", NAMESPACE, + f"--replicas={target}") + if r.returncode != 0: + result["error"] = f"Scale failed: {r.stderr}" + print(json.dumps(result, indent=2)) + return 1 + + # Wait for new machines to be ready + print("Waiting for new machines to provision...", file=sys.stderr) + deadline = time.time() + 1500 + while time.time() < deadline: + r = run_oc("get", "machineset", ms_name, "-n", NAMESPACE, + "-o", "jsonpath={.status.readyReplicas}") + ready = int(r.stdout.strip()) if r.returncode == 0 and r.stdout.strip() else 0 + if ready >= target: + break + print(f" Ready: {ready}/{target}", file=sys.stderr) + time.sleep(30) + + # Count nodes + r = run_kubectl("get", "nodes", "-l", + f"machine.openshift.io/cluster-api-machineset={ms_name}", + "--no-headers") + nodes = [l for l in r.stdout.strip().split("\n") if l.strip()] if r.returncode == 0 else [] + + result["scaled_to"] = target + result["nodes_after"] = len(nodes) + result["success"] = len(nodes) >= target + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/machineset/verify_machine_api.py b/isvctl/configs/stubs/openshift/machineset/verify_machine_api.py new file mode 100644 index 00000000..fc7e18e0 --- /dev/null +++ b/isvctl/configs/stubs/openshift/machineset/verify_machine_api.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Verify Machine API is configured with Carbide provider. + +Checks that the Machine API operator is running and the Carbide +infrastructure provider is configured. + +Output: {"success": true, "provider_configured": true, "machine_api_ready": true} +""" + +import json +import subprocess +import sys +from typing import Any + + +def run_oc(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["oc"] + list(args), capture_output=True, text=True, timeout=60) + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "provider_configured": False, + "machine_api_ready": False, + } + + # Check Machine API operator + r = run_oc("get", "clusteroperator", "machine-api", + "-o", "jsonpath={.status.conditions[?(@.type=='Available')].status}") + result["machine_api_ready"] = r.returncode == 0 and r.stdout.strip() == "True" + + if not result["machine_api_ready"]: + result["error"] = "Machine API operator not available" + print(json.dumps(result, indent=2)) + return 1 + + # Check infrastructure platform type + r = run_oc("get", "infrastructure", "cluster", + "-o", "jsonpath={.status.platformStatus.type}") + platform_type = r.stdout.strip() if r.returncode == 0 else "" + result["platform_type"] = platform_type + + # Check for existing MachineSets (proves Machine API is functional) + r = run_oc("get", "machinesets", "-n", "openshift-machine-api", "--no-headers") + existing_machinesets = len(r.stdout.strip().split("\n")) if r.stdout.strip() else 0 + result["existing_machinesets"] = existing_machinesets + + # Check if Carbide provider CRDs or configuration exist + r = run_oc("get", "machines", "-n", "openshift-machine-api", "--no-headers") + result["existing_machines"] = len(r.stdout.strip().split("\n")) if r.stdout.strip() else 0 + + result["provider_configured"] = result["machine_api_ready"] + result["success"] = result["provider_configured"] + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/network/multus_test.py b/isvctl/configs/stubs/openshift/network/multus_test.py new file mode 100644 index 00000000..36dae96b --- /dev/null +++ b/isvctl/configs/stubs/openshift/network/multus_test.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Test Multus secondary network interface attachment. + +Validates: + - Multus pods are running in the openshift-multus namespace + - A NetworkAttachmentDefinition can be created (macvlan/bridge) + - A pod with a secondary network annotation receives multiple + network interfaces + +Environment: + K8S_NAMESPACE: Test namespace (default: ncp-network-validation) + +Output schema: generic (fields: multus_ready, + secondary_interface_attached) +""" + +import json +import os +import subprocess +import sys +import time +from typing import Any + + +def run_cmd(cmd: str, *args: str, **kwargs) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [cmd] + list(args), capture_output=True, text=True, + timeout=kwargs.get("timeout", 120), + ) + + +def wait_for_pod_ready(namespace: str, name: str, timeout: int = 120) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + r = run_cmd( + "kubectl", "get", "pod", name, "-n", namespace, + "-o", "jsonpath={.status.phase}", + ) + if r.returncode == 0 and r.stdout.strip() == "Running": + return True + time.sleep(5) + return False + + +def cleanup(namespace: str) -> None: + """Remove test pod and NetworkAttachmentDefinition.""" + run_cmd("kubectl", "delete", "pod", "multus-test-pod", "-n", namespace, + "--ignore-not-found", "--grace-period=0", "--force") + run_cmd("kubectl", "delete", "net-attach-def", "ncp-bridge-net", + "-n", namespace, "--ignore-not-found") + + +def main() -> int: + namespace = os.environ.get("K8S_NAMESPACE", "ncp-network-validation") + + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "multus_ready": False, + "secondary_interface_attached": False, + } + + try: + # Check Multus pods running in openshift-multus namespace + r = run_cmd("kubectl", "get", "pods", "-n", "openshift-multus", + "-o", "json") + multus_running = False + if r.returncode == 0: + pods = json.loads(r.stdout).get("items", []) + for pod in pods: + name = pod.get("metadata", {}).get("name", "") + phase = pod.get("status", {}).get("phase", "") + if "multus" in name and phase == "Running": + multus_running = True + break + + result["multus_ready"] = multus_running + if not multus_running: + result["error"] = "Multus pods not found running in openshift-multus namespace." + print(json.dumps(result, indent=2)) + return 1 + + print("Multus pods are running.", file=sys.stderr) + + # Clean up any leftovers + cleanup(namespace) + + # Create NetworkAttachmentDefinition (bridge CNI) + nad_manifest = json.dumps({ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": { + "name": "ncp-bridge-net", + "namespace": namespace, + }, + "spec": { + "config": json.dumps({ + "cniVersion": "0.3.1", + "type": "bridge", + "bridge": "ncp-br0", + "ipam": { + "type": "host-local", + "subnet": "10.200.0.0/24", + "rangeStart": "10.200.0.10", + "rangeEnd": "10.200.0.250", + }, + }), + }, + }) + proc = subprocess.run( + ["kubectl", "apply", "-f", "-"], + input=nad_manifest, capture_output=True, text=True, timeout=120, + ) + if proc.returncode != 0: + result["error"] = ( + f"Failed to create NetworkAttachmentDefinition: {proc.stderr}" + ) + print(json.dumps(result, indent=2)) + return 1 + + print("NetworkAttachmentDefinition created.", file=sys.stderr) + + # Create pod with secondary network annotation + pod_manifest = json.dumps({ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "multus-test-pod", + "namespace": namespace, + "annotations": { + "k8s.v1.cni.cncf.io/networks": "ncp-bridge-net", + }, + }, + "spec": { + "containers": [{ + "name": "test", + "image": "registry.access.redhat.com/ubi9/ubi-minimal:latest", + "command": ["sh", "-c", "sleep 3600"], + }], + "restartPolicy": "Never", + }, + }) + proc = subprocess.run( + ["kubectl", "apply", "-f", "-"], + input=pod_manifest, capture_output=True, text=True, timeout=120, + ) + if proc.returncode != 0: + result["error"] = f"Failed to create test pod: {proc.stderr}" + cleanup(namespace) + print(json.dumps(result, indent=2)) + return 1 + + # Wait for pod readiness + print("Waiting for Multus test pod...", file=sys.stderr) + if not wait_for_pod_ready(namespace, "multus-test-pod"): + result["error"] = "Multus test pod did not become ready" + cleanup(namespace) + print(json.dumps(result, indent=2)) + return 1 + + # Verify pod has multiple interfaces + r = run_cmd( + "kubectl", "exec", "multus-test-pod", "-n", namespace, "--", + "sh", "-c", "ip -o link show | wc -l", + ) + iface_count = 0 + if r.returncode == 0: + try: + iface_count = int(r.stdout.strip()) + except ValueError: + pass + + # At least 3 interfaces expected: lo, eth0 (primary), net1 (secondary) + secondary_attached = iface_count >= 3 + result["secondary_interface_attached"] = secondary_attached + + if secondary_attached: + print( + f"Pod has {iface_count} interfaces (secondary attached).", + file=sys.stderr, + ) + else: + print( + f"Pod has {iface_count} interface(s); expected >= 3.", + file=sys.stderr, + ) + + # Also check the network-status annotation for confirmation + r = run_cmd( + "kubectl", "get", "pod", "multus-test-pod", "-n", namespace, + "-o", "jsonpath={.metadata.annotations.k8s\\.v1\\.cni\\.cncf\\.io/network-status}", + ) + if r.returncode == 0 and r.stdout.strip(): + try: + net_status = json.loads(r.stdout.strip()) + if len(net_status) >= 2: + secondary_attached = True + result["secondary_interface_attached"] = True + print( + f"Network status confirms {len(net_status)} networks.", + file=sys.stderr, + ) + except (json.JSONDecodeError, TypeError): + pass + + result["success"] = multus_running and secondary_attached + + except Exception as e: + result["error"] = str(e) + finally: + cleanup(namespace) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/network/nccl_multinode_test.py b/isvctl/configs/stubs/openshift/network/nccl_multinode_test.py new file mode 100644 index 00000000..860e6171 --- /dev/null +++ b/isvctl/configs/stubs/openshift/network/nccl_multinode_test.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Multi-node NCCL AllReduce over RDMA. + +Requires at least 2 GPU nodes with RDMA. Creates an NCCL +all_reduce_perf Job across 2 nodes using nvcr.io/nvidia/pytorch +image and parses bus bandwidth from the output. + +Falls back from NCCL_NET=IB to NCCL_NET=Socket if IB is +unavailable. If fewer than 2 GPU nodes exist, the test succeeds +with skipped=true. + +Environment: + K8S_NAMESPACE: Test namespace (default: ncp-network-validation) + NCCL_IMAGE: Container image (default: nvcr.io/nvidia/pytorch:24.01-py3) + +Output schema: generic (fields: status, nodes_used, + bus_bandwidth_gbps, nccl_net) +""" + +import json +import os +import re +import subprocess +import sys +import time +from typing import Any + + +def run_cmd(cmd: str, *args: str, **kwargs) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [cmd] + list(args), capture_output=True, text=True, + timeout=kwargs.get("timeout", 120), + ) + + +def get_gpu_nodes() -> list[str]: + """Return list of node names that have nvidia.com/gpu allocatable.""" + r = run_cmd("kubectl", "get", "nodes", "-o", "json") + if r.returncode != 0: + return [] + nodes = json.loads(r.stdout).get("items", []) + gpu_nodes = [] + for node in nodes: + allocatable = node.get("status", {}).get("allocatable", {}) + gpus = int(allocatable.get("nvidia.com/gpu", "0")) + if gpus > 0: + name = node.get("metadata", {}).get("name", "") + if name: + gpu_nodes.append(name) + return gpu_nodes + + +def cleanup(namespace: str) -> None: + run_cmd("kubectl", "delete", "job", "nccl-allreduce", "-n", namespace, + "--ignore-not-found") + # Also delete any pods created by the job + run_cmd("kubectl", "delete", "pods", "-n", namespace, + "-l", "job-name=nccl-allreduce", "--ignore-not-found", + "--grace-period=0", "--force") + + +def main() -> int: + namespace = os.environ.get("K8S_NAMESPACE", "ncp-network-validation") + nccl_image = os.environ.get( + "NCCL_IMAGE", "nvcr.io/nvidia/pytorch:24.01-py3" + ) + + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "status": "failed", + "nodes_used": 0, + "bus_bandwidth_gbps": 0.0, + "nccl_net": "", + } + + try: + gpu_nodes = get_gpu_nodes() + if len(gpu_nodes) < 2: + print( + f"Only {len(gpu_nodes)} GPU node(s) found; need at least 2. " + "Skipping multi-node NCCL test.", + file=sys.stderr, + ) + result["success"] = True + result["status"] = "skipped" + result["skipped"] = True + result["nodes_used"] = len(gpu_nodes) + print(json.dumps(result, indent=2)) + return 0 + + print( + f"Found {len(gpu_nodes)} GPU nodes: {', '.join(gpu_nodes)}", + file=sys.stderr, + ) + + # Clean up any leftovers + cleanup(namespace) + + # Determine NCCL_NET: try IB first + # Check if RDMA resources are available + has_rdma = False + r = run_cmd("kubectl", "get", "nodes", "-o", "json") + if r.returncode == 0: + nodes = json.loads(r.stdout).get("items", []) + for node in nodes: + allocatable = node.get("status", {}).get("allocatable", {}) + if int(allocatable.get("rdma/rdma_shared_device_a", "0")) > 0: + has_rdma = True + break + + nccl_net = "IB" if has_rdma else "Socket" + result["nccl_net"] = nccl_net + + # Build resource requests + resource_limits: dict[str, str] = {"nvidia.com/gpu": "1"} + if has_rdma: + resource_limits["rdma/rdma_shared_device_a"] = "1" + + # NCCL allreduce test command + nccl_cmd = ( + "apt-get update -qq && apt-get install -y -qq openssh-server > /dev/null 2>&1; " + "all_reduce_perf -b 8 -e 128M -f 2 -g 1 2>&1; " + "exit 0" + ) + + # Create a Job with 2 completions (one per node) + # Using indexed completion mode for multi-node + job_manifest = json.dumps({ + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "name": "nccl-allreduce", + "namespace": namespace, + }, + "spec": { + "completions": 1, + "parallelism": 1, + "backoffLimit": 2, + "activeDeadlineSeconds": 600, + "template": { + "metadata": { + "labels": {"app": "nccl-allreduce"}, + }, + "spec": { + "containers": [{ + "name": "nccl", + "image": nccl_image, + "command": ["bash", "-c", nccl_cmd], + "env": [ + {"name": "NCCL_NET", "value": nccl_net}, + {"name": "NCCL_DEBUG", "value": "INFO"}, + {"name": "NCCL_IB_DISABLE", + "value": "0" if has_rdma else "1"}, + ], + "resources": { + "limits": resource_limits, + }, + "securityContext": { + "capabilities": { + "add": ["IPC_LOCK"], + }, + }, + }], + "restartPolicy": "Never", + "affinity": { + "podAntiAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": [{ + "labelSelector": { + "matchLabels": {"app": "nccl-allreduce"}, + }, + "topologyKey": "kubernetes.io/hostname", + }], + }, + }, + }, + }, + }, + }) + + proc = subprocess.run( + ["kubectl", "apply", "-f", "-"], + input=job_manifest, capture_output=True, text=True, timeout=120, + ) + if proc.returncode != 0: + result["error"] = f"Failed to create NCCL job: {proc.stderr}" + print(json.dumps(result, indent=2)) + return 1 + + print("NCCL AllReduce job created. Waiting for completion...", + file=sys.stderr) + + # Wait for job completion + deadline = time.time() + 600 + job_done = False + while time.time() < deadline: + r = run_cmd( + "kubectl", "get", "job", "nccl-allreduce", "-n", namespace, + "-o", "json", + ) + if r.returncode == 0: + job = json.loads(r.stdout) + conditions = job.get("status", {}).get("conditions", []) + for cond in conditions: + if cond.get("type") == "Complete" and cond.get("status") == "True": + job_done = True + break + if cond.get("type") == "Failed" and cond.get("status") == "True": + result["error"] = "NCCL job failed" + job_done = True + break + if job_done: + break + time.sleep(10) + + if not job_done: + result["error"] = "NCCL job timed out after 600s" + cleanup(namespace) + print(json.dumps(result, indent=2)) + return 1 + + # Get pod logs + r = run_cmd( + "kubectl", "logs", "-n", namespace, + "-l", "job-name=nccl-allreduce", "--tail=200", + timeout=60, + ) + logs = r.stdout if r.returncode == 0 else "" + + # Determine which node ran the pod + r = run_cmd( + "kubectl", "get", "pods", "-n", namespace, + "-l", "job-name=nccl-allreduce", + "-o", "jsonpath={.items[*].spec.nodeName}", + ) + nodes_used_names = set(r.stdout.strip().split()) if r.returncode == 0 else set() + result["nodes_used"] = len(nodes_used_names) + + # Parse bus bandwidth from all_reduce_perf output + # Format: size count type redop root time algbw busbw #wrong + bus_bw = 0.0 + for line in logs.split("\n"): + # Match lines with numeric data from all_reduce_perf + m = re.match( + r"\s*\d+\s+\d+\s+\S+\s+\S+\s+\S+\s+[\d.]+\s+[\d.]+\s+([\d.]+)", + line, + ) + if m: + try: + bw = float(m.group(1)) + if bw > bus_bw: + bus_bw = bw + except ValueError: + pass + + result["bus_bandwidth_gbps"] = round(bus_bw, 2) + + if "error" not in result: + result["success"] = True + result["status"] = "passed" + print( + f"NCCL AllReduce completed. Peak bus BW: {bus_bw:.2f} GB/s " + f"(NCCL_NET={nccl_net})", + file=sys.stderr, + ) + else: + print(f"NCCL job failed: {result.get('error')}", file=sys.stderr) + + except Exception as e: + result["error"] = str(e) + finally: + cleanup(namespace) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/network/network_operator_check.py b/isvctl/configs/stubs/openshift/network/network_operator_check.py new file mode 100644 index 00000000..2f5a9fcb --- /dev/null +++ b/isvctl/configs/stubs/openshift/network/network_operator_check.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Check NVIDIA Network Operator is installed and healthy. + +Validates the following components: + - NicClusterPolicy CR exists + - MOFED driver pods are running in the nvidia-network-operator namespace + - RDMA shared device plugin pods are running + - GPU nodes expose rdma/rdma_shared_device_a allocatable resource + +Environment: + NETWORK_OPERATOR_NAMESPACE: Namespace for the operator + (default: nvidia-network-operator) + +Output schema: generic (fields: mofed_ready, mofed_pods, + rdma_device_plugin_ready, rdma_devices_per_node) +""" + +import json +import subprocess +import sys +from typing import Any + + +def run_cmd(cmd: str, *args: str, **kwargs) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [cmd] + list(args), capture_output=True, text=True, + timeout=kwargs.get("timeout", 120), + ) + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "mofed_ready": False, + "mofed_pods": 0, + "rdma_device_plugin_ready": False, + "rdma_devices_per_node": 0, + } + + try: + # Check NicClusterPolicy CR exists + r = run_cmd("kubectl", "get", "nicclusterpolicies.mellanox.com", "-A", + "-o", "json") + if r.returncode != 0: + result["error"] = ( + "NicClusterPolicy CRD not found. " + "NVIDIA Network Operator may not be installed." + ) + print(json.dumps(result, indent=2)) + return 1 + + policies = json.loads(r.stdout) + items = policies.get("items", []) + if not items: + result["error"] = "No NicClusterPolicy resources found." + print(json.dumps(result, indent=2)) + return 1 + + print(f"Found {len(items)} NicClusterPolicy resource(s).", file=sys.stderr) + + # Check MOFED driver pods running + ns = "nvidia-network-operator" + r = run_cmd("kubectl", "get", "pods", "-n", ns, + "-l", "app=mofed", + "-o", "json") + mofed_pods = 0 + if r.returncode == 0: + pods = json.loads(r.stdout).get("items", []) + for pod in pods: + phase = pod.get("status", {}).get("phase", "") + if phase == "Running": + mofed_pods += 1 + + # Fallback: search by name pattern if label selector found nothing + if mofed_pods == 0: + r = run_cmd("kubectl", "get", "pods", "-n", ns, "-o", "json") + if r.returncode == 0: + pods = json.loads(r.stdout).get("items", []) + for pod in pods: + name = pod.get("metadata", {}).get("name", "") + phase = pod.get("status", {}).get("phase", "") + if "mofed" in name and phase == "Running": + mofed_pods += 1 + + result["mofed_pods"] = mofed_pods + result["mofed_ready"] = mofed_pods > 0 + if mofed_pods > 0: + print(f"MOFED driver pods running: {mofed_pods}", file=sys.stderr) + else: + print("No MOFED driver pods found running.", file=sys.stderr) + + # Check RDMA shared device plugin pods + rdma_dp_ready = False + r = run_cmd("kubectl", "get", "pods", "-n", ns, "-o", "json") + if r.returncode == 0: + pods = json.loads(r.stdout).get("items", []) + for pod in pods: + name = pod.get("metadata", {}).get("name", "") + phase = pod.get("status", {}).get("phase", "") + if "rdma-shared" in name and phase == "Running": + rdma_dp_ready = True + break + + result["rdma_device_plugin_ready"] = rdma_dp_ready + if rdma_dp_ready: + print("RDMA shared device plugin pods found.", file=sys.stderr) + else: + print("No RDMA shared device plugin pods found.", file=sys.stderr) + + # Check rdma/rdma_shared_device_a resource on GPU nodes + r = run_cmd("kubectl", "get", "nodes", "-o", "json") + rdma_per_node = 0 + if r.returncode == 0: + nodes = json.loads(r.stdout).get("items", []) + for node in nodes: + allocatable = node.get("status", {}).get("allocatable", {}) + # Check for GPU presence + gpus = int(allocatable.get("nvidia.com/gpu", "0")) + if gpus > 0: + rdma_count = int( + allocatable.get("rdma/rdma_shared_device_a", "0") + ) + if rdma_count > rdma_per_node: + rdma_per_node = rdma_count + + result["rdma_devices_per_node"] = rdma_per_node + if rdma_per_node > 0: + print( + f"RDMA devices per GPU node: {rdma_per_node}", file=sys.stderr + ) + + # Success if at least the NicClusterPolicy exists and MOFED is running + result["success"] = result["mofed_ready"] + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/network/network_policy_test.py b/isvctl/configs/stubs/openshift/network/network_policy_test.py new file mode 100644 index 00000000..4bc394d0 --- /dev/null +++ b/isvctl/configs/stubs/openshift/network/network_policy_test.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Test NetworkPolicy enforcement across namespaces. + +Creates pods in two namespaces, applies a deny NetworkPolicy, +verifies cross-namespace traffic is blocked, then removes the +policy and verifies traffic is allowed again. + +Environment: + K8S_NAMESPACE: Primary namespace (default: ncp-network-validation) + +Output schema: generic (fields: policy_enforced, traffic_blocked, + traffic_allowed_after_delete) +""" + +import json +import os +import subprocess +import sys +import time +from typing import Any + + +def run_cmd(cmd: str, *args: str, **kwargs) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [cmd] + list(args), capture_output=True, text=True, + timeout=kwargs.get("timeout", 120), + ) + + +def wait_for_pod_ready(namespace: str, name: str, timeout: int = 120) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + r = run_cmd( + "kubectl", "get", "pod", name, "-n", namespace, + "-o", "jsonpath={.status.phase}", + ) + if r.returncode == 0 and r.stdout.strip() == "Running": + return True + time.sleep(5) + return False + + +def cleanup(ns1: str, ns2: str) -> None: + """Remove test pods and network policy.""" + run_cmd("kubectl", "delete", "pod", "netpol-server", "-n", ns1, + "--ignore-not-found", "--grace-period=0", "--force") + run_cmd("kubectl", "delete", "pod", "netpol-client", "-n", ns2, + "--ignore-not-found", "--grace-period=0", "--force") + run_cmd("kubectl", "delete", "networkpolicy", "deny-cross-ns", + "-n", ns1, "--ignore-not-found") + + +def main() -> int: + ns1 = os.environ.get("K8S_NAMESPACE", "ncp-network-validation") + ns2 = "ncp-network-validation-2" + + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "policy_enforced": False, + "traffic_blocked": False, + "traffic_allowed_after_delete": False, + } + + try: + # Clean up any leftovers + cleanup(ns1, ns2) + + # Create server pod in ns1 with a simple HTTP listener + server_yaml = json.dumps({ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "netpol-server", + "namespace": ns1, + "labels": {"app": "netpol-server"}, + }, + "spec": { + "containers": [{ + "name": "server", + "image": "registry.access.redhat.com/ubi9/ubi-minimal:latest", + "command": [ + "sh", "-c", + "python3 -m http.server 8080 || " + "while true; do echo -e 'HTTP/1.1 200 OK\\r\\n\\r\\nOK' " + "| nc -l -p 8080 -q 1; done", + ], + "ports": [{"containerPort": 8080}], + }], + "restartPolicy": "Never", + }, + }) + r = run_cmd("kubectl", "apply", "-f", "-", input=server_yaml) + if r.returncode != 0: + # Try with stdin via subprocess directly + proc = subprocess.run( + ["kubectl", "apply", "-f", "-"], + input=server_yaml, capture_output=True, text=True, timeout=120, + ) + if proc.returncode != 0: + result["error"] = f"Failed to create server pod: {proc.stderr}" + print(json.dumps(result, indent=2)) + return 1 + + # Create client pod in ns2 + client_yaml = json.dumps({ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "netpol-client", + "namespace": ns2, + }, + "spec": { + "containers": [{ + "name": "client", + "image": "registry.access.redhat.com/ubi9/ubi-minimal:latest", + "command": ["sh", "-c", "sleep 3600"], + }], + "restartPolicy": "Never", + }, + }) + subprocess.run( + ["kubectl", "apply", "-f", "-"], + input=client_yaml, capture_output=True, text=True, timeout=120, + ) + + # Wait for pods to be ready + print("Waiting for pods to become ready...", file=sys.stderr) + if not wait_for_pod_ready(ns1, "netpol-server"): + result["error"] = "Server pod did not become ready" + cleanup(ns1, ns2) + print(json.dumps(result, indent=2)) + return 1 + + if not wait_for_pod_ready(ns2, "netpol-client"): + result["error"] = "Client pod did not become ready" + cleanup(ns1, ns2) + print(json.dumps(result, indent=2)) + return 1 + + # Get server pod IP + r = run_cmd( + "kubectl", "get", "pod", "netpol-server", "-n", ns1, + "-o", "jsonpath={.status.podIP}", + ) + server_ip = r.stdout.strip() + if not server_ip: + result["error"] = "Could not get server pod IP" + cleanup(ns1, ns2) + print(json.dumps(result, indent=2)) + return 1 + + print(f"Server pod IP: {server_ip}", file=sys.stderr) + + # Apply NetworkPolicy to deny all ingress from other namespaces + netpol_yaml = json.dumps({ + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "name": "deny-cross-ns", + "namespace": ns1, + }, + "spec": { + "podSelector": {"matchLabels": {"app": "netpol-server"}}, + "policyTypes": ["Ingress"], + "ingress": [{ + "from": [{ + "podSelector": {}, + }], + }], + }, + }) + subprocess.run( + ["kubectl", "apply", "-f", "-"], + input=netpol_yaml, capture_output=True, text=True, timeout=120, + ) + + # Allow a moment for the policy to be applied + time.sleep(5) + + # Test that traffic from ns2 is blocked + print("Testing cross-namespace traffic (should be blocked)...", + file=sys.stderr) + r = run_cmd( + "kubectl", "exec", "netpol-client", "-n", ns2, "--", + "sh", "-c", + f"wget -q -O /dev/null --timeout=5 http://{server_ip}:8080/ 2>&1" + " && echo REACHABLE || echo BLOCKED", + timeout=30, + ) + traffic_blocked = "BLOCKED" in r.stdout or r.returncode != 0 + result["traffic_blocked"] = traffic_blocked + result["policy_enforced"] = traffic_blocked + + if traffic_blocked: + print("Cross-namespace traffic correctly blocked.", file=sys.stderr) + else: + print("WARNING: Traffic was NOT blocked by policy.", + file=sys.stderr) + + # Delete the NetworkPolicy + run_cmd("kubectl", "delete", "networkpolicy", "deny-cross-ns", + "-n", ns1) + time.sleep(5) + + # Test that traffic is now allowed + print("Testing cross-namespace traffic (should be allowed)...", + file=sys.stderr) + r = run_cmd( + "kubectl", "exec", "netpol-client", "-n", ns2, "--", + "sh", "-c", + f"wget -q -O /dev/null --timeout=10 http://{server_ip}:8080/ 2>&1" + " && echo REACHABLE || echo BLOCKED", + timeout=30, + ) + traffic_allowed = "REACHABLE" in r.stdout + result["traffic_allowed_after_delete"] = traffic_allowed + + if traffic_allowed: + print("Traffic correctly allowed after policy removal.", + file=sys.stderr) + else: + print("WARNING: Traffic still blocked after policy removal.", + file=sys.stderr) + + result["success"] = traffic_blocked and traffic_allowed + + except Exception as e: + result["error"] = str(e) + finally: + cleanup(ns1, ns2) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/network/rdma_test.py b/isvctl/configs/stubs/openshift/network/rdma_test.py new file mode 100644 index 00000000..d743ff51 --- /dev/null +++ b/isvctl/configs/stubs/openshift/network/rdma_test.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Test RDMA device availability in GPU pods. + +Creates a pod requesting rdma/rdma_shared_device_a, verifies +RDMA devices are accessible (/dev/infiniband or ibv_devinfo), +and checks GPUDirect RDMA readiness (nvidia_peermem module). + +If no RDMA devices are available on the cluster, the test +succeeds with rdma_available=false and skipped=true. + +Environment: + K8S_NAMESPACE: Test namespace (default: ncp-network-validation) + +Output schema: generic (fields: rdma_available, gpudirect_ready) +""" + +import json +import os +import subprocess +import sys +import time +from typing import Any + + +def run_cmd(cmd: str, *args: str, **kwargs) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [cmd] + list(args), capture_output=True, text=True, + timeout=kwargs.get("timeout", 120), + ) + + +def wait_for_pod_ready(namespace: str, name: str, timeout: int = 180) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + r = run_cmd( + "kubectl", "get", "pod", name, "-n", namespace, + "-o", "jsonpath={.status.phase}", + ) + if r.returncode == 0 and r.stdout.strip() == "Running": + return True + # Check for unschedulable (e.g. no RDMA resource available) + r2 = run_cmd( + "kubectl", "get", "pod", name, "-n", namespace, + "-o", "jsonpath={.status.conditions[?(@.type=='PodScheduled')].reason}", + ) + if r2.returncode == 0 and r2.stdout.strip() == "Unschedulable": + return False + time.sleep(5) + return False + + +def cleanup(namespace: str) -> None: + run_cmd("kubectl", "delete", "pod", "rdma-test-pod", "-n", namespace, + "--ignore-not-found", "--grace-period=0", "--force") + + +def main() -> int: + namespace = os.environ.get("K8S_NAMESPACE", "ncp-network-validation") + + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "rdma_available": False, + "gpudirect_ready": False, + } + + try: + # Check if any node has rdma/rdma_shared_device_a + r = run_cmd("kubectl", "get", "nodes", "-o", "json") + has_rdma_resource = False + if r.returncode == 0: + nodes = json.loads(r.stdout).get("items", []) + for node in nodes: + allocatable = node.get("status", {}).get("allocatable", {}) + rdma_count = int( + allocatable.get("rdma/rdma_shared_device_a", "0") + ) + if rdma_count > 0: + has_rdma_resource = True + break + + if not has_rdma_resource: + print( + "No rdma/rdma_shared_device_a resources on any node. " + "Skipping RDMA test.", + file=sys.stderr, + ) + result["success"] = True + result["rdma_available"] = False + result["skipped"] = True + print(json.dumps(result, indent=2)) + return 0 + + # Clean up any leftovers + cleanup(namespace) + + # Create a pod requesting RDMA resource + pod_manifest = json.dumps({ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "rdma-test-pod", + "namespace": namespace, + }, + "spec": { + "containers": [{ + "name": "rdma", + "image": "registry.access.redhat.com/ubi9/ubi-minimal:latest", + "command": ["sh", "-c", "sleep 3600"], + "resources": { + "limits": { + "rdma/rdma_shared_device_a": "1", + }, + }, + "securityContext": { + "capabilities": { + "add": ["IPC_LOCK"], + }, + }, + }], + "restartPolicy": "Never", + }, + }) + proc = subprocess.run( + ["kubectl", "apply", "-f", "-"], + input=pod_manifest, capture_output=True, text=True, timeout=120, + ) + if proc.returncode != 0: + result["error"] = f"Failed to create RDMA test pod: {proc.stderr}" + print(json.dumps(result, indent=2)) + return 1 + + print("Waiting for RDMA test pod...", file=sys.stderr) + if not wait_for_pod_ready(namespace, "rdma-test-pod"): + # Pod could not be scheduled (no RDMA resources available) + r = run_cmd( + "kubectl", "get", "pod", "rdma-test-pod", "-n", namespace, + "-o", "jsonpath={.status.conditions[?(@.type=='PodScheduled')].message}", + ) + msg = r.stdout.strip() if r.returncode == 0 else "unknown" + print(f"RDMA test pod unschedulable: {msg}", file=sys.stderr) + result["success"] = True + result["rdma_available"] = False + result["skipped"] = True + cleanup(namespace) + print(json.dumps(result, indent=2)) + return 0 + + # Check for RDMA devices inside the pod + rdma_available = False + + # Check /dev/infiniband + r = run_cmd( + "kubectl", "exec", "rdma-test-pod", "-n", namespace, "--", + "sh", "-c", "ls /dev/infiniband/ 2>/dev/null && echo FOUND || echo NOTFOUND", + ) + if r.returncode == 0 and "FOUND" in r.stdout: + rdma_available = True + print("RDMA devices found in /dev/infiniband/.", file=sys.stderr) + + # Try ibv_devinfo as fallback + if not rdma_available: + r = run_cmd( + "kubectl", "exec", "rdma-test-pod", "-n", namespace, "--", + "sh", "-c", "ibv_devinfo 2>/dev/null && echo FOUND || echo NOTFOUND", + ) + if r.returncode == 0 and "FOUND" in r.stdout and "hca_id" in r.stdout: + rdma_available = True + print("ibv_devinfo reports RDMA devices.", file=sys.stderr) + + result["rdma_available"] = rdma_available + + # Check GPUDirect RDMA (nvidia_peermem module) + gpudirect_ready = False + # Check on the node where the pod is running + r = run_cmd( + "kubectl", "get", "pod", "rdma-test-pod", "-n", namespace, + "-o", "jsonpath={.spec.nodeName}", + ) + if r.returncode == 0 and r.stdout.strip(): + node_name = r.stdout.strip() + # Use debug pod to check kernel module + r = run_cmd( + "oc", "debug", f"node/{node_name}", "--", + "chroot", "/host", "lsmod", + timeout=60, + ) + if r.returncode == 0 and "nvidia_peermem" in r.stdout: + gpudirect_ready = True + print("nvidia_peermem module loaded.", file=sys.stderr) + else: + print("nvidia_peermem module not loaded.", file=sys.stderr) + + result["gpudirect_ready"] = gpudirect_ready + result["success"] = rdma_available + + except Exception as e: + result["error"] = str(e) + finally: + cleanup(namespace) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/network/setup_namespaces.py b/isvctl/configs/stubs/openshift/network/setup_namespaces.py new file mode 100644 index 00000000..348d39bf --- /dev/null +++ b/isvctl/configs/stubs/openshift/network/setup_namespaces.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Create test namespaces for GPU network validation. + +Creates the ncp-network-validation and ncp-network-validation-2 +namespaces used by subsequent network validation tests. +All steps are idempotent -- safe to run repeatedly. + +Environment: + K8S_NAMESPACE: Primary namespace (default: ncp-network-validation) + +Output schema: generic (fields: namespaces) +""" + +import json +import subprocess +import sys +from typing import Any + + +def run_cmd(cmd: str, *args: str, **kwargs) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [cmd] + list(args), capture_output=True, text=True, + timeout=kwargs.get("timeout", 120), + ) + + +def main() -> int: + namespaces = ["ncp-network-validation", "ncp-network-validation-2"] + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "namespaces": [], + } + + try: + created = [] + for ns in namespaces: + # Check if namespace already exists + r = run_cmd("kubectl", "get", "namespace", ns, "--no-headers") + if r.returncode == 0: + print(f"Namespace '{ns}' already exists.", file=sys.stderr) + created.append(ns) + continue + + r = run_cmd("kubectl", "create", "namespace", ns) + if r.returncode != 0: + result["error"] = f"Failed to create namespace '{ns}': {r.stderr}" + print(json.dumps(result, indent=2)) + return 1 + + print(f"Namespace '{ns}' created.", file=sys.stderr) + created.append(ns) + + result["namespaces"] = created + result["success"] = True + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/network/sriov_check.py b/isvctl/configs/stubs/openshift/network/sriov_check.py new file mode 100644 index 00000000..c29a9ef6 --- /dev/null +++ b/isvctl/configs/stubs/openshift/network/sriov_check.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Check SR-IOV Operator and VF provisioning on OpenShift. + +Validates the following: + - SriovNetworkNodeState CRD exists + - SR-IOV operator pods are running + - Virtual Functions (VFs) are configured on GPU nodes + - SriovNetwork resources exist + +Environment: + SRIOV_NAMESPACE: SR-IOV operator namespace + (default: openshift-sriov-network-operator) + +Output schema: generic (fields: sriov_operator_ready, vf_count, + sriov_networks) +""" + +import json +import subprocess +import sys +from typing import Any + + +def run_cmd(cmd: str, *args: str, **kwargs) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [cmd] + list(args), capture_output=True, text=True, + timeout=kwargs.get("timeout", 120), + ) + + +def main() -> int: + sriov_ns = "openshift-sriov-network-operator" + + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "sriov_operator_ready": False, + "vf_count": 0, + "sriov_networks": [], + } + + try: + # Check SriovNetworkNodeState CRD exists + r = run_cmd("oc", "get", "sriovnetworknodestates", + "-n", sriov_ns, "-o", "json") + if r.returncode != 0: + result["error"] = ( + "SriovNetworkNodeState CRD not found. " + "SR-IOV Operator may not be installed." + ) + print(json.dumps(result, indent=2)) + return 1 + + node_states = json.loads(r.stdout).get("items", []) + print( + f"Found {len(node_states)} SriovNetworkNodeState resource(s).", + file=sys.stderr, + ) + + # Check SR-IOV operator pods running + r = run_cmd("kubectl", "get", "pods", "-n", sriov_ns, "-o", "json") + operator_running = False + if r.returncode == 0: + pods = json.loads(r.stdout).get("items", []) + for pod in pods: + name = pod.get("metadata", {}).get("name", "") + phase = pod.get("status", {}).get("phase", "") + if "sriov-network-operator" in name and phase == "Running": + operator_running = True + break + + result["sriov_operator_ready"] = operator_running + if operator_running: + print("SR-IOV operator pod is running.", file=sys.stderr) + else: + print("SR-IOV operator pod not found running.", file=sys.stderr) + + # Count VFs configured on GPU nodes + total_vfs = 0 + for state in node_states: + interfaces = ( + state.get("status", {}).get("interfaces", []) + ) + for iface in interfaces: + num_vfs = iface.get("numVfs", 0) + total_vfs += num_vfs + + result["vf_count"] = total_vfs + if total_vfs > 0: + print(f"Total VFs configured: {total_vfs}", file=sys.stderr) + else: + print("No VFs configured on any node.", file=sys.stderr) + + # Check SriovNetwork resources + r = run_cmd("oc", "get", "sriovnetworks", "-n", sriov_ns, "-o", "json") + sriov_networks: list[str] = [] + if r.returncode == 0: + items = json.loads(r.stdout).get("items", []) + for item in items: + name = item.get("metadata", {}).get("name", "") + if name: + sriov_networks.append(name) + + result["sriov_networks"] = sriov_networks + if sriov_networks: + print( + f"SriovNetwork resources: {', '.join(sriov_networks)}", + file=sys.stderr, + ) + else: + print("No SriovNetwork resources found.", file=sys.stderr) + + # Success if operator is running (VFs and networks may still + # be in the process of being configured) + result["success"] = operator_running + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/network/teardown.py b/isvctl/configs/stubs/openshift/network/teardown.py new file mode 100644 index 00000000..258c0641 --- /dev/null +++ b/isvctl/configs/stubs/openshift/network/teardown.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Delete test namespaces and remaining network validation resources. + +Removes ncp-network-validation and ncp-network-validation-2 +namespaces along with all resources they contain. + +Output schema: teardown +""" + +import json +import subprocess +import sys +import time +from typing import Any + + +def run_cmd(cmd: str, *args: str, **kwargs) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [cmd] + list(args), capture_output=True, text=True, + timeout=kwargs.get("timeout", 120), + ) + + +def main() -> int: + namespaces = ["ncp-network-validation", "ncp-network-validation-2"] + + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "resources_deleted": [], + } + + try: + for ns in namespaces: + # Check if namespace exists + r = run_cmd("kubectl", "get", "namespace", ns, "--no-headers") + if r.returncode != 0: + print(f"Namespace '{ns}' does not exist, skipping.", + file=sys.stderr) + continue + + # Delete all pods first (in case any are stuck) + r = run_cmd("kubectl", "delete", "pods", "--all", "-n", ns, + "--grace-period=0", "--force", "--ignore-not-found") + if r.returncode == 0: + print(f"Deleted pods in namespace '{ns}'.", file=sys.stderr) + + # Delete all jobs + run_cmd("kubectl", "delete", "jobs", "--all", "-n", ns, + "--ignore-not-found") + + # Delete all network policies + run_cmd("kubectl", "delete", "networkpolicies", "--all", "-n", ns, + "--ignore-not-found") + + # Delete all NetworkAttachmentDefinitions + run_cmd("kubectl", "delete", "net-attach-def", "--all", "-n", ns, + "--ignore-not-found") + + # Delete the namespace + r = run_cmd("kubectl", "delete", "namespace", ns, + "--ignore-not-found", timeout=120) + if r.returncode == 0: + result["resources_deleted"].append(f"namespace/{ns}") + print(f"Namespace '{ns}' deleted.", file=sys.stderr) + else: + print( + f"Warning: failed to delete namespace '{ns}': {r.stderr}", + file=sys.stderr, + ) + + # Wait briefly for namespace termination + deadline = time.time() + 60 + while time.time() < deadline: + all_gone = True + for ns in namespaces: + r = run_cmd("kubectl", "get", "namespace", ns, "--no-headers") + if r.returncode == 0: + all_gone = False + break + if all_gone: + break + time.sleep(5) + + result["success"] = True + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/security/compliance_scan.py b/isvctl/configs/stubs/openshift/security/compliance_scan.py new file mode 100644 index 00000000..51fbd8e0 --- /dev/null +++ b/isvctl/configs/stubs/openshift/security/compliance_scan.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Check Compliance Operator scan results. + +Verifies the Compliance Operator is installed and checks for any +existing scan results. Does NOT trigger new scans. +""" + +import json +import subprocess +import sys +from typing import Any + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "operator_installed": False, + "scans_found": 0, + "compliant": 0, + "non_compliant": 0, + } + + # Check if Compliance Operator is installed + r = subprocess.run( + ["kubectl", "get", "csv", "-n", "openshift-compliance", "--no-headers"], + capture_output=True, text=True, timeout=30, + ) + if r.returncode == 0 and "compliance-operator" in r.stdout: + result["operator_installed"] = True + else: + result["success"] = True + result["info"] = "Compliance Operator not installed — skipping scan check" + print(json.dumps(result, indent=2)) + return 0 + + # Check ComplianceSuite results + r = subprocess.run( + ["kubectl", "get", "compliancesuites", "-n", "openshift-compliance", "-o", "json"], + capture_output=True, text=True, timeout=30, + ) + if r.returncode == 0: + data = json.loads(r.stdout) + for suite in data.get("items", []): + result["scans_found"] += 1 + phase = suite.get("status", {}).get("phase", "") + if phase == "DONE": + res = suite.get("status", {}).get("result", "") + if res == "Compliant": + result["compliant"] += 1 + elif res == "NonCompliant": + result["non_compliant"] += 1 + + # Check ComplianceCheckResult summary + r = subprocess.run( + ["kubectl", "get", "compliancecheckresults", "-n", "openshift-compliance", + "--no-headers"], + capture_output=True, text=True, timeout=30, + ) + if r.returncode == 0 and r.stdout.strip(): + lines = r.stdout.strip().split("\n") + result["total_check_results"] = len(lines) + result["pass_count"] = sum(1 for l in lines if "PASS" in l) + result["fail_count"] = sum(1 for l in lines if "FAIL" in l) + + result["success"] = True + + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/security/fips_check.py b/isvctl/configs/stubs/openshift/security/fips_check.py new file mode 100644 index 00000000..5ce43273 --- /dev/null +++ b/isvctl/configs/stubs/openshift/security/fips_check.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Check FIPS mode status on the cluster.""" + +import json +import subprocess +import sys +from typing import Any + + +def main() -> int: + result: dict[str, Any] = {"success": False, "platform": "openshift", "fips_enabled": False} + + # Check cluster-wide FIPS status via configmap + r = subprocess.run( + ["oc", "get", "configmap", "cluster-config-v1", "-n", "kube-system", + "-o", "jsonpath={.data.install-config}"], + capture_output=True, text=True, timeout=30, + ) + if r.returncode == 0 and "fips: true" in r.stdout.lower(): + result["fips_enabled"] = True + result["source"] = "install-config" + else: + # Fallback: check a node directly + r = subprocess.run( + ["kubectl", "get", "nodes", "-o", "jsonpath={.items[0].metadata.name}"], + capture_output=True, text=True, timeout=30, + ) + if r.returncode == 0 and r.stdout.strip(): + node = r.stdout.strip() + r = subprocess.run( + ["oc", "debug", f"node/{node}", "--", "chroot", "/host", + "cat", "/proc/sys/crypto/fips_enabled"], + capture_output=True, text=True, timeout=60, + ) + if r.returncode == 0 and r.stdout.strip() == "1": + result["fips_enabled"] = True + result["source"] = "kernel" + + # FIPS is optional — report status but don't fail if disabled + result["success"] = True + if not result["fips_enabled"]: + result["info"] = "FIPS mode is not enabled (optional for non-regulated environments)" + + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/security/secure_boot_check.py b/isvctl/configs/stubs/openshift/security/secure_boot_check.py new file mode 100644 index 00000000..39ce958e --- /dev/null +++ b/isvctl/configs/stubs/openshift/security/secure_boot_check.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Check Secure Boot is enabled on all nodes.""" + +import json +import subprocess +import sys +from typing import Any + + +def main() -> int: + result: dict[str, Any] = {"success": False, "platform": "openshift", "nodes_checked": 0, "all_secure_boot": False} + + r = subprocess.run(["kubectl", "get", "nodes", "-o", "jsonpath={.items[*].metadata.name}"], + capture_output=True, text=True, timeout=30) + if r.returncode != 0: + result["error"] = f"Failed to list nodes: {r.stderr}" + print(json.dumps(result, indent=2)) + return 1 + + nodes = r.stdout.strip().split() + enabled = 0 + node_results = {} + + for node in nodes: + r = subprocess.run( + ["oc", "debug", f"node/{node}", "--", "chroot", "/host", + "mokutil", "--sb-state"], + capture_output=True, text=True, timeout=60, + ) + output = r.stdout.strip() if r.returncode == 0 else "" + sb_enabled = "SecureBoot enabled" in output + node_results[node] = "enabled" if sb_enabled else output or "unknown" + if sb_enabled: + enabled += 1 + + result["nodes_checked"] = len(nodes) + result["secure_boot_count"] = enabled + result["all_secure_boot"] = enabled == len(nodes) + result["node_results"] = node_results + result["success"] = result["all_secure_boot"] + + if not result["success"] and enabled == 0: + # Secure Boot may not be available on all hardware — warn, don't fail + result["success"] = True + result["warning"] = "Secure Boot not detected — may not be supported by hardware" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/security/selinux_check.py b/isvctl/configs/stubs/openshift/security/selinux_check.py new file mode 100644 index 00000000..319369aa --- /dev/null +++ b/isvctl/configs/stubs/openshift/security/selinux_check.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Check SELinux is in Enforcing mode on all nodes.""" + +import json +import subprocess +import sys +from typing import Any + + +def main() -> int: + result: dict[str, Any] = {"success": False, "platform": "openshift", "nodes_checked": 0, "all_enforcing": False} + + r = subprocess.run(["kubectl", "get", "nodes", "-o", "jsonpath={.items[*].metadata.name}"], + capture_output=True, text=True, timeout=30) + if r.returncode != 0: + result["error"] = f"Failed to list nodes: {r.stderr}" + print(json.dumps(result, indent=2)) + return 1 + + nodes = r.stdout.strip().split() + enforcing = 0 + node_results = {} + + for node in nodes: + r = subprocess.run( + ["oc", "debug", f"node/{node}", "--", "chroot", "/host", "getenforce"], + capture_output=True, text=True, timeout=60, + ) + status = r.stdout.strip() if r.returncode == 0 else "Unknown" + node_results[node] = status + if status == "Enforcing": + enforcing += 1 + + result["nodes_checked"] = len(nodes) + result["enforcing_count"] = enforcing + result["all_enforcing"] = enforcing == len(nodes) + result["node_results"] = node_results + result["success"] = result["all_enforcing"] + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/storage/deploy_odf.py b/isvctl/configs/stubs/openshift/storage/deploy_odf.py new file mode 100644 index 00000000..ad1f4309 --- /dev/null +++ b/isvctl/configs/stubs/openshift/storage/deploy_odf.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Deploy ODF operator and create StorageCluster (idempotent). + +Installs ODF operator from OperatorHub, discovers local NVMe disks +via LocalVolumeDiscovery, and creates a StorageCluster using them. + +Environment: + ODF_NAMESPACE: ODF namespace (default: openshift-storage) + ODF_DISK_COUNT: Min NVMe disks per node (default: 1) + ODF_STORAGE_NODES: Comma-separated node names (default: auto-detect) + +Output: {"success": true, "odf_ready": true, "disks_discovered": N, ...} +""" + +import json +import os +import subprocess +import sys +import time +from typing import Any + + +def run_oc(*args: str, input_data: str | None = None) -> subprocess.CompletedProcess[str]: + cmd = ["oc"] + list(args) + return subprocess.run(cmd, capture_output=True, text=True, input=input_data, timeout=120) + + +def run_kubectl(*args: str) -> subprocess.CompletedProcess[str]: + cmd = ["kubectl"] + list(args) + return subprocess.run(cmd, capture_output=True, text=True, timeout=120) + + +def wait_for_csv(namespace: str, name_prefix: str, timeout: int = 300) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + r = run_oc("get", "csv", "-n", namespace, + "-o", "jsonpath={range .items[*]}{.metadata.name},{.status.phase}{'\\n'}{end}") + if r.returncode == 0: + for line in r.stdout.strip().split("\n"): + if "," in line: + name, phase = line.rsplit(",", 1) + if name.startswith(name_prefix) and phase == "Succeeded": + return True + time.sleep(10) + return False + + +def main() -> int: + namespace = os.environ.get("ODF_NAMESPACE", "openshift-storage") + min_disks = int(os.environ.get("ODF_DISK_COUNT", "1")) + storage_nodes_str = os.environ.get("ODF_STORAGE_NODES", "") + + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "odf_ready": False, + "disks_discovered": 0, + "namespace": namespace, + } + + try: + # Check if ODF is already deployed + r = run_oc("get", "storagecluster", "-n", namespace, "--no-headers") + if r.returncode == 0 and r.stdout.strip(): + print("StorageCluster already exists.", file=sys.stderr) + # Verify it's ready + r = run_oc("get", "storagecluster", "-n", namespace, + "-o", "jsonpath={.items[0].status.phase}") + if r.stdout.strip() == "Ready": + result["odf_ready"] = True + result["disks_discovered"] = min_disks + result["success"] = True + print(json.dumps(result, indent=2)) + return 0 + + # Create namespace + run_oc("create", "namespace", namespace) + + # Label namespace for ODF + run_oc("label", "namespace", namespace, + "openshift.io/cluster-monitoring=true", "--overwrite") + + # Create OperatorGroup + og_yaml = f""" +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: openshift-storage-operatorgroup + namespace: {namespace} +spec: + targetNamespaces: + - {namespace} +""" + run_oc("apply", "-f", "-", input_data=og_yaml) + + # Create Subscription for ODF operator + sub_yaml = f""" +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: odf-operator + namespace: {namespace} +spec: + channel: stable-4.18 + name: odf-operator + source: redhat-operators + sourceNamespace: openshift-marketplace + installPlanApproval: Automatic +""" + run_oc("apply", "-f", "-", input_data=sub_yaml) + + print("Waiting for ODF operator CSV...", file=sys.stderr) + if not wait_for_csv(namespace, "odf-operator", timeout=600): + raise RuntimeError("ODF operator CSV did not reach Succeeded") + + # Auto-detect storage nodes if not specified + if storage_nodes_str: + storage_nodes = [n.strip() for n in storage_nodes_str.split(",")] + else: + # Use all worker/schedulable nodes + r = run_kubectl("get", "nodes", + "-l", "node-role.kubernetes.io/worker=", + "-o", "jsonpath={.items[*].metadata.name}") + if r.returncode == 0 and r.stdout.strip(): + storage_nodes = r.stdout.strip().split() + else: + r = run_kubectl("get", "nodes", "--no-headers", + "-o", "custom-columns=NAME:.metadata.name") + storage_nodes = [n.strip() for n in r.stdout.strip().split("\n") + if n.strip()] + + # Label nodes for ODF + for node in storage_nodes: + run_oc("label", "node", node, + "cluster.ocs.openshift.io/openshift-storage=", "--overwrite") + + result["storage_nodes"] = storage_nodes + print(f"Storage nodes: {storage_nodes}", file=sys.stderr) + + # Create LocalVolumeDiscovery to find NVMe disks + lvd_yaml = f""" +apiVersion: local.storage.openshift.io/v1alpha1 +kind: LocalVolumeDiscovery +metadata: + name: auto-discover-devices + namespace: {namespace} +spec: + nodeSelector: + nodeSelectorTerms: + - matchExpressions: + - key: cluster.ocs.openshift.io/openshift-storage + operator: Exists +""" + run_oc("apply", "-f", "-", input_data=lvd_yaml) + + # Wait for disk discovery + print("Waiting for NVMe disk discovery...", file=sys.stderr) + time.sleep(30) + + # Create StorageCluster + sc_yaml = f""" +apiVersion: ocs.openshift.io/v1 +kind: StorageCluster +metadata: + name: ocs-storagecluster + namespace: {namespace} +spec: + manageNodes: false + monDataDirHostPath: /var/lib/rook + storageDeviceSets: + - name: ocs-deviceset + count: {len(storage_nodes)} + dataPVCTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: localblock + volumeMode: Block + portable: false + replica: {min(len(storage_nodes), 3)} +""" + run_oc("apply", "-f", "-", input_data=sc_yaml) + + # Wait for StorageCluster to become Ready + print("Waiting for StorageCluster to become Ready...", file=sys.stderr) + deadline = time.time() + 600 + while time.time() < deadline: + r = run_oc("get", "storagecluster", "ocs-storagecluster", "-n", namespace, + "-o", "jsonpath={.status.phase}") + if r.returncode == 0 and r.stdout.strip() == "Ready": + result["odf_ready"] = True + break + time.sleep(15) + + if not result["odf_ready"]: + # Accept Progressing as partial success + r = run_oc("get", "storagecluster", "ocs-storagecluster", "-n", namespace, + "-o", "jsonpath={.status.phase}") + phase = r.stdout.strip() if r.returncode == 0 else "Unknown" + print(f"StorageCluster phase: {phase}", file=sys.stderr) + if phase in ("Progressing", "Ready"): + result["odf_ready"] = True + + result["disks_discovered"] = len(storage_nodes) * min_disks + result["success"] = result["odf_ready"] + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/storage/teardown.py b/isvctl/configs/stubs/openshift/storage/teardown.py new file mode 100644 index 00000000..0c639e4a --- /dev/null +++ b/isvctl/configs/stubs/openshift/storage/teardown.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Clean up storage test resources. + +Deletes test PVCs, pods, and the test namespace. Does NOT remove +ODF or the StorageCluster — those stay deployed for other tests. + +Output: {"success": true, "resources_deleted": [...]} +""" + +import json +import os +import subprocess +import sys +from typing import Any + + +NAMESPACE = os.environ.get("K8S_NAMESPACE", "ncp-storage-validation") + + +def run_kubectl(*args: str) -> subprocess.CompletedProcess[str]: + cmd = ["kubectl"] + list(args) + return subprocess.run(cmd, capture_output=True, text=True, timeout=60) + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "resources_deleted": [], + } + + # Delete test PVCs + for pvc in ["ncp-test-rbd", "ncp-test-cephfs"]: + r = run_kubectl("delete", "pvc", pvc, "-n", NAMESPACE, "--ignore-not-found") + if r.returncode == 0: + result["resources_deleted"].append(f"pvc/{pvc}") + + # Delete test namespace + r = run_kubectl("delete", "namespace", NAMESPACE, "--ignore-not-found") + if r.returncode == 0: + result["resources_deleted"].append(f"namespace/{NAMESPACE}") + + result["success"] = True + result["message"] = "Test resources cleaned up; ODF stays deployed" + + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/storage/test_pod_mount.py b/isvctl/configs/stubs/openshift/storage/test_pod_mount.py new file mode 100644 index 00000000..89df3237 --- /dev/null +++ b/isvctl/configs/stubs/openshift/storage/test_pod_mount.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Test pod mount with ODF-backed PVC. + +Creates a pod that mounts the RBD PVC from test_pvc_binding, writes +data, reads it back, and verifies integrity. + +Output: {"success": true, "write_ok": true, "read_ok": true, "data_matches": true} +""" + +import json +import os +import subprocess +import sys +import time +from typing import Any + + +NAMESPACE = os.environ.get("K8S_NAMESPACE", "ncp-storage-validation") + + +def run_kubectl(*args: str, input_data: str | None = None) -> subprocess.CompletedProcess[str]: + cmd = ["kubectl"] + list(args) + return subprocess.run(cmd, capture_output=True, text=True, input=input_data, timeout=120) + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "write_ok": False, + "read_ok": False, + "data_matches": False, + } + + pod_name = "ncp-storage-test-pod" + + try: + # Check if the RBD PVC exists (created by test_pvc_binding) + r = run_kubectl("get", "pvc", "ncp-test-rbd", "-n", NAMESPACE, "--no-headers") + if r.returncode != 0: + result["error"] = "RBD PVC ncp-test-rbd not found. Run test_pvc_binding first." + print(json.dumps(result, indent=2)) + return 1 + + # Create test pod mounting the PVC + pod_yaml = f""" +apiVersion: v1 +kind: Pod +metadata: + name: {pod_name} + namespace: {NAMESPACE} +spec: + containers: + - name: test + image: registry.access.redhat.com/ubi9/ubi-minimal:latest + command: ["sleep", "300"] + volumeMounts: + - name: storage + mountPath: /data + volumes: + - name: storage + persistentVolumeClaim: + claimName: ncp-test-rbd + restartPolicy: Never +""" + run_kubectl("delete", "pod", pod_name, "-n", NAMESPACE, "--ignore-not-found") + run_kubectl("apply", "-f", "-", input_data=pod_yaml) + + # Wait for pod to be Running + print("Waiting for test pod...", file=sys.stderr) + deadline = time.time() + 120 + while time.time() < deadline: + r = run_kubectl("get", "pod", pod_name, "-n", NAMESPACE, + "-o", "jsonpath={.status.phase}") + if r.returncode == 0 and r.stdout.strip() == "Running": + break + time.sleep(5) + else: + result["error"] = "Test pod did not reach Running state" + print(json.dumps(result, indent=2)) + return 1 + + # Write test data + test_data = "ncp-storage-validation-ok" + r = run_kubectl("exec", pod_name, "-n", NAMESPACE, "--", + "sh", "-c", f"echo '{test_data}' > /data/test.txt") + result["write_ok"] = r.returncode == 0 + + # Read test data back + r = run_kubectl("exec", pod_name, "-n", NAMESPACE, "--", + "cat", "/data/test.txt") + result["read_ok"] = r.returncode == 0 + result["data_matches"] = r.stdout.strip() == test_data + + result["success"] = result["write_ok"] and result["read_ok"] and result["data_matches"] + + except Exception as e: + result["error"] = str(e) + finally: + # Clean up pod + run_kubectl("delete", "pod", pod_name, "-n", NAMESPACE, "--ignore-not-found") + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/storage/test_pvc_binding.py b/isvctl/configs/stubs/openshift/storage/test_pvc_binding.py new file mode 100644 index 00000000..3d6e90c1 --- /dev/null +++ b/isvctl/configs/stubs/openshift/storage/test_pvc_binding.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Test PVC binding with ODF StorageClasses. + +Creates PVCs for both CephRBD and CephFS StorageClasses and verifies +they reach Bound state. + +Output: {"success": true, "rbd_pvc_bound": true, "cephfs_pvc_bound": true} +""" + +import json +import os +import subprocess +import sys +import time +from typing import Any + + +NAMESPACE = os.environ.get("K8S_NAMESPACE", "ncp-storage-validation") + + +def run_kubectl(*args: str, input_data: str | None = None) -> subprocess.CompletedProcess[str]: + cmd = ["kubectl"] + list(args) + return subprocess.run(cmd, capture_output=True, text=True, input=input_data, timeout=120) + + +def create_and_wait_pvc(name: str, storage_class: str, timeout: int = 120) -> bool: + """Create a PVC and wait for it to bind.""" + pvc_yaml = f""" +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {name} + namespace: {NAMESPACE} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: {storage_class} +""" + run_kubectl("apply", "-f", "-", input_data=pvc_yaml) + + deadline = time.time() + timeout + while time.time() < deadline: + r = run_kubectl("get", "pvc", name, "-n", NAMESPACE, + "-o", "jsonpath={.status.phase}") + if r.returncode == 0 and r.stdout.strip() == "Bound": + return True + time.sleep(5) + return False + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "rbd_pvc_bound": False, + "cephfs_pvc_bound": False, + } + + try: + # Ensure namespace exists + run_kubectl("create", "namespace", NAMESPACE) + + # Detect available StorageClass names + r = run_kubectl("get", "storageclass", "-o", "jsonpath={.items[*].metadata.name}") + classes = r.stdout.strip().split() if r.returncode == 0 else [] + + # Find the RBD StorageClass + rbd_class = "" + for name in ["ocs-storagecluster-ceph-rbd", "ocs-external-storagecluster-ceph-rbd"]: + if name in classes: + rbd_class = name + break + + # Find the CephFS StorageClass + cephfs_class = "" + for name in ["ocs-storagecluster-cephfs", "ocs-external-storagecluster-cephfs"]: + if name in classes: + cephfs_class = name + break + + # Test RBD PVC + if rbd_class: + print(f"Creating RBD PVC with {rbd_class}...", file=sys.stderr) + result["rbd_pvc_bound"] = create_and_wait_pvc("ncp-test-rbd", rbd_class) + print(f" RBD PVC bound: {result['rbd_pvc_bound']}", file=sys.stderr) + else: + print("No CephRBD StorageClass found, skipping.", file=sys.stderr) + + # Test CephFS PVC + if cephfs_class: + print(f"Creating CephFS PVC with {cephfs_class}...", file=sys.stderr) + result["cephfs_pvc_bound"] = create_and_wait_pvc("ncp-test-cephfs", cephfs_class) + print(f" CephFS PVC bound: {result['cephfs_pvc_bound']}", file=sys.stderr) + else: + print("No CephFS StorageClass found, skipping.", file=sys.stderr) + + result["success"] = result["rbd_pvc_bound"] and result["cephfs_pvc_bound"] + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/storage/verify_storage_classes.py b/isvctl/configs/stubs/openshift/storage/verify_storage_classes.py new file mode 100644 index 00000000..8eccf799 --- /dev/null +++ b/isvctl/configs/stubs/openshift/storage/verify_storage_classes.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Verify ODF StorageClasses are provisioned. + +Checks that CephRBD and CephFS StorageClasses exist after ODF deployment. + +Output: {"success": true, "rbd_class_exists": true, "cephfs_class_exists": true, ...} +""" + +import json +import subprocess +import sys +from typing import Any + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "openshift", + "rbd_class_exists": False, + "cephfs_class_exists": False, + "storage_classes": [], + } + + try: + r = subprocess.run( + ["kubectl", "get", "storageclass", "-o", + "jsonpath={.items[*].metadata.name}"], + capture_output=True, text=True, timeout=30, + ) + if r.returncode != 0: + result["error"] = f"Failed to list StorageClasses: {r.stderr}" + print(json.dumps(result, indent=2)) + return 1 + + classes = r.stdout.strip().split() + result["storage_classes"] = classes + + # Check for OCS/ODF StorageClasses + rbd_names = ["ocs-storagecluster-ceph-rbd", "ocs-external-storagecluster-ceph-rbd"] + cephfs_names = ["ocs-storagecluster-cephfs", "ocs-external-storagecluster-cephfs"] + + result["rbd_class_exists"] = any(n in classes for n in rbd_names) + result["cephfs_class_exists"] = any(n in classes for n in cephfs_names) + + result["success"] = result["rbd_class_exists"] and result["cephfs_class_exists"] + + if not result["success"]: + missing = [] + if not result["rbd_class_exists"]: + missing.append("CephRBD") + if not result["cephfs_class_exists"]: + missing.append("CephFS") + result["error"] = f"Missing StorageClasses: {', '.join(missing)}" + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/vm/deploy_nim.py b/isvctl/configs/stubs/openshift/vm/deploy_nim.py new file mode 100644 index 00000000..e9eeae37 --- /dev/null +++ b/isvctl/configs/stubs/openshift/vm/deploy_nim.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Deploy NIM inside the GPU VM (optional, skipped by default).""" + +import json +import sys +from typing import Any + + +def main() -> int: + result: dict[str, Any] = { + "success": True, + "platform": "vm", + "skipped": True, + "message": "NIM deploy in VM skipped (enable via config)", + } + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/vm/launch_instance.py b/isvctl/configs/stubs/openshift/vm/launch_instance.py new file mode 100644 index 00000000..7b3b0ff2 --- /dev/null +++ b/isvctl/configs/stubs/openshift/vm/launch_instance.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Create a GPU-enabled VirtualMachine via KubeVirt. + +Creates a VirtualMachine CR with GPU passthrough (VFIO), waits for +it to reach Running state, and outputs connection info for SSH checks. + +Environment: + VM_NAMESPACE: Namespace (default: ncp-vm-validation) + VM_GPU_COUNT: GPUs to passthrough (default: 1) + VM_GPU_DEVICE: GPU device name (default: auto-detect from node labels) + VM_MEMORY: VM memory (default: 16Gi) + VM_CPUS: VM CPU cores (default: 8) + VM_DISK_SIZE: Root disk size (default: 50Gi) + VM_IMAGE_URL: Cloud image URL (RHEL/CentOS/Fedora qcow2) + VM_SSH_PUBKEY: SSH public key for cloud-user access + VM_SSH_KEY: SSH private key path (for framework SSH checks) +""" + +import json +import os +import subprocess +import sys +import time +from typing import Any + +NAMESPACE = os.environ.get("VM_NAMESPACE", "ncp-vm-validation") +VM_NAME = "ncp-gpu-vm" + + +def run_kubectl(*args: str, input_data: str | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, + input=input_data, timeout=120) + + +def detect_gpu_device() -> str: + """Auto-detect GPU device name from node labels.""" + r = run_kubectl("get", "nodes", "-l", "nvidia.com/gpu.present=true", + "-o", "jsonpath={.items[0].metadata.labels.nvidia\\.com/gpu\\.product}") + if r.returncode == 0 and r.stdout.strip(): + return r.stdout.strip().replace(" ", "_") + return "nvidia.com/GH200" + + +def main() -> int: + gpu_count = int(os.environ.get("VM_GPU_COUNT", "1")) + gpu_device = os.environ.get("VM_GPU_DEVICE", "") or detect_gpu_device() + memory = os.environ.get("VM_MEMORY", "16Gi") + cpus = int(os.environ.get("VM_CPUS", "8")) + disk_size = os.environ.get("VM_DISK_SIZE", "50Gi") + image_url = os.environ.get("VM_IMAGE_URL", "") + ssh_pubkey = os.environ.get("VM_SSH_PUBKEY", "") + ssh_key = os.environ.get("VM_SSH_KEY", "") + + result: dict[str, Any] = { + "success": False, + "platform": "vm", + "instance_id": VM_NAME, + } + + if not image_url: + result["error"] = "VM_IMAGE_URL is required (path to qcow2 cloud image)" + print(json.dumps(result, indent=2)) + return 1 + + try: + # Create namespace + run_kubectl("create", "namespace", NAMESPACE) + + # Build GPU devices list + gpus_yaml = "" + for i in range(gpu_count): + gpus_yaml += f""" + - deviceName: {gpu_device} + name: gpu-{i}""" + + # Cloud-init user data + cloud_init = """ +#cloud-config +user: cloud-user +ssh_authorized_keys: [] +""" + if ssh_pubkey: + cloud_init = f""" +#cloud-config +user: cloud-user +ssh_authorized_keys: + - {ssh_pubkey} +""" + + # VirtualMachine CR + vm_yaml = f""" +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + name: {VM_NAME} + namespace: {NAMESPACE} +spec: + running: true + template: + spec: + domain: + cpu: + cores: {cpus} + memory: + guest: {memory} + devices: + disks: + - name: rootdisk + disk: + bus: virtio + - name: cloudinitdisk + disk: + bus: virtio + gpus:{gpus_yaml} + interfaces: + - name: default + masquerade: {{}} + resources: + requests: + memory: {memory} + networks: + - name: default + pod: {{}} + volumes: + - name: rootdisk + dataVolume: + name: {VM_NAME}-rootdisk + - name: cloudinitdisk + cloudInitNoCloud: + userData: | + {cloud_init.strip()} + dataVolumeTemplates: + - metadata: + name: {VM_NAME}-rootdisk + spec: + source: + http: + url: "{image_url}" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {disk_size} +""" + # Delete existing VM if any + run_kubectl("delete", "vm", VM_NAME, "-n", NAMESPACE, "--ignore-not-found") + time.sleep(5) + + r = run_kubectl("apply", "-f", "-", input_data=vm_yaml) + if r.returncode != 0: + result["error"] = f"Failed to create VM: {r.stderr}" + print(json.dumps(result, indent=2)) + return 1 + + # Wait for VMI to be Running + print(f"Waiting for VM {VM_NAME} to start...", file=sys.stderr) + deadline = time.time() + 600 + while time.time() < deadline: + r = run_kubectl("get", "vmi", VM_NAME, "-n", NAMESPACE, + "-o", "jsonpath={.status.phase}") + if r.returncode == 0 and r.stdout.strip() == "Running": + break + time.sleep(10) + else: + result["error"] = "VM did not reach Running state" + print(json.dumps(result, indent=2)) + return 1 + + # Get VM IP + r = run_kubectl("get", "vmi", VM_NAME, "-n", NAMESPACE, + "-o", "jsonpath={.status.interfaces[0].ipAddress}") + vm_ip = r.stdout.strip() if r.returncode == 0 else "" + + if not vm_ip: + # Fallback: try pod IP + r = run_kubectl("get", "vmi", VM_NAME, "-n", NAMESPACE, + "-o", "jsonpath={.status.nodeName}") + vm_ip = "pending" + + # Wait for SSH + if vm_ip and vm_ip != "pending": + print(f"Waiting for SSH on {vm_ip}...", file=sys.stderr) + deadline = time.time() + 180 + while time.time() < deadline: + r = subprocess.run( + ["ssh", "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=5", + "-o", "BatchMode=yes"] + + (["-i", ssh_key] if ssh_key else []) + + [f"cloud-user@{vm_ip}", "true"], + capture_output=True, text=True, timeout=10, + ) + if r.returncode == 0: + print("SSH ready.", file=sys.stderr) + break + time.sleep(10) + + result["public_ip"] = vm_ip + result["private_ip"] = vm_ip + result["state"] = "running" + result["ssh_user"] = "cloud-user" + result["key_file"] = ssh_key + result["vpc_id"] = NAMESPACE + result["success"] = True + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/vm/list_instances.py b/isvctl/configs/stubs/openshift/vm/list_instances.py new file mode 100644 index 00000000..3aec3bc7 --- /dev/null +++ b/isvctl/configs/stubs/openshift/vm/list_instances.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""List VirtualMachineInstances in the test namespace.""" + +import json +import os +import subprocess +import sys +from typing import Any + +NAMESPACE = os.environ.get("VM_NAMESPACE", "ncp-vm-validation") +TARGET = "ncp-gpu-vm" + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "vm", + "instances": [], + "count": 0, + "found_target": False, + "target_instance": TARGET, + } + + r = subprocess.run( + ["kubectl", "get", "vmi", "-n", NAMESPACE, "-o", "json"], + capture_output=True, text=True, timeout=30, + ) + if r.returncode != 0: + result["error"] = f"Failed to list VMIs: {r.stderr}" + print(json.dumps(result, indent=2)) + return 1 + + data = json.loads(r.stdout) + for item in data.get("items", []): + name = item["metadata"]["name"] + phase = item.get("status", {}).get("phase", "Unknown") + ips = item.get("status", {}).get("interfaces", []) + ip = ips[0].get("ipAddress", "") if ips else "" + result["instances"].append({ + "instance_id": name, + "state": phase.lower(), + "public_ip": ip, + "private_ip": ip, + }) + if name == TARGET: + result["found_target"] = True + + result["count"] = len(result["instances"]) + result["success"] = True + + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/vm/reboot_instance.py b/isvctl/configs/stubs/openshift/vm/reboot_instance.py new file mode 100644 index 00000000..0e37f006 --- /dev/null +++ b/isvctl/configs/stubs/openshift/vm/reboot_instance.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Restart the GPU VM and verify it comes back.""" + +import json +import os +import subprocess +import sys +import time +from typing import Any + +NAMESPACE = os.environ.get("VM_NAMESPACE", "ncp-vm-validation") +VM_NAME = "ncp-gpu-vm" + + +def run_kubectl(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, timeout=120) + + +def main() -> int: + ssh_key = os.environ.get("VM_SSH_KEY", "") + + result: dict[str, Any] = { + "success": False, + "platform": "vm", + "instance_id": VM_NAME, + "reboot_initiated": False, + "ssh_ready": False, + } + + try: + # Restart via virtctl or VMI delete + r = subprocess.run(["virtctl", "restart", VM_NAME, "-n", NAMESPACE], + capture_output=True, text=True, timeout=30) + if r.returncode != 0: + # Fallback: delete VMI to trigger restart + run_kubectl("delete", "vmi", VM_NAME, "-n", NAMESPACE) + + result["reboot_initiated"] = True + print("Restart initiated, waiting for VMI to come back...", file=sys.stderr) + time.sleep(15) + + # Wait for Running + deadline = time.time() + 300 + while time.time() < deadline: + r = run_kubectl("get", "vmi", VM_NAME, "-n", NAMESPACE, + "-o", "jsonpath={.status.phase}") + if r.returncode == 0 and r.stdout.strip() == "Running": + break + time.sleep(10) + + # Get IP + r = run_kubectl("get", "vmi", VM_NAME, "-n", NAMESPACE, + "-o", "jsonpath={.status.interfaces[0].ipAddress}") + vm_ip = r.stdout.strip() if r.returncode == 0 else "" + + # Wait for SSH + if vm_ip: + print(f"Waiting for SSH on {vm_ip}...", file=sys.stderr) + deadline = time.time() + 180 + while time.time() < deadline: + r = subprocess.run( + ["ssh", "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=5", + "-o", "BatchMode=yes"] + + (["-i", ssh_key] if ssh_key else []) + + [f"cloud-user@{vm_ip}", "true"], + capture_output=True, text=True, timeout=10, + ) + if r.returncode == 0: + result["ssh_ready"] = True + break + time.sleep(10) + + # Get uptime + if result["ssh_ready"]: + r = subprocess.run( + ["ssh", "-o", "StrictHostKeyChecking=no", "-o", "BatchMode=yes"] + + (["-i", ssh_key] if ssh_key else []) + + [f"cloud-user@{vm_ip}", "cat", "/proc/uptime"], + capture_output=True, text=True, timeout=10, + ) + if r.returncode == 0: + uptime = float(r.stdout.strip().split()[0]) + result["uptime_seconds"] = int(uptime) + + result["public_ip"] = vm_ip + result["private_ip"] = vm_ip + result["state"] = "running" + result["ssh_user"] = "cloud-user" + result["key_file"] = ssh_key + result["ssh_connectivity"] = result["ssh_ready"] + result["success"] = result["reboot_initiated"] and result["ssh_ready"] + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/openshift/vm/teardown.py b/isvctl/configs/stubs/openshift/vm/teardown.py new file mode 100644 index 00000000..ae29fa8c --- /dev/null +++ b/isvctl/configs/stubs/openshift/vm/teardown.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Delete the GPU VM and test namespace.""" + +import json +import os +import subprocess +import sys +import time +from typing import Any + +NAMESPACE = os.environ.get("VM_NAMESPACE", "ncp-vm-validation") +VM_NAME = "ncp-gpu-vm" + + +def run_kubectl(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["kubectl"] + list(args), capture_output=True, text=True, timeout=120) + + +def main() -> int: + result: dict[str, Any] = { + "success": False, + "platform": "vm", + "resources_deleted": [], + } + + # Delete VM + r = run_kubectl("delete", "vm", VM_NAME, "-n", NAMESPACE, "--ignore-not-found") + if r.returncode == 0: + result["resources_deleted"].append(f"vm/{VM_NAME}") + + # Wait for VMI to terminate + deadline = time.time() + 120 + while time.time() < deadline: + r = run_kubectl("get", "vmi", VM_NAME, "-n", NAMESPACE, "--no-headers") + if r.returncode != 0 or not r.stdout.strip(): + break + time.sleep(10) + + # Delete DataVolume + run_kubectl("delete", "dv", f"{VM_NAME}-rootdisk", "-n", NAMESPACE, "--ignore-not-found") + result["resources_deleted"].append(f"dv/{VM_NAME}-rootdisk") + + # Delete namespace + run_kubectl("delete", "namespace", NAMESPACE, "--ignore-not-found") + result["resources_deleted"].append(f"namespace/{NAMESPACE}") + + result["success"] = True + + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/templates/README.md b/isvctl/configs/templates/README.md index a6dfc946..1cba1b6a 100644 --- a/isvctl/configs/templates/README.md +++ b/isvctl/configs/templates/README.md @@ -1,38 +1,100 @@ # Validation Templates -Provider-agnostic templates for ISV Lab validation tests. Copy a template, implement the stub scripts for your platform, and run. +Provider-agnostic validation templates for ISV Lab tests. Templates define **what to validate** (checks); provider configs define **how to provision** (stubs). ## How It Works ```text -┌──────────────────┐ ┌──────────────────────┐ ┌────────────────────┐ -│ YAML Config │─────▶│ Your Stub Scripts │─────▶│ Validations │ -│ (steps + args) │ │ (call your API) │ │ (check JSON) │ -│ │ │ │ │ │ -│ You configure │ │ YOU IMPLEMENT THESE │ │ Already provided │ -│ step names, │ │ Output JSON to │ │ StepSuccessCheck, │ -│ args, timeouts │ │ stdout │ │ FieldExistsCheck │ -└──────────────────┘ └──────────────────────┘ └────────────────────┘ +┌──────────────────┐ ┌──────────────────────┐ ┌────────────────────┐ +│ Template YAML │ │ Provider YAML │ │ Validations │ +│ (validations) │ │ (commands + stubs) │ │ (check JSON) │ +│ │ │ │ │ │ +│ WHAT to check │ + │ HOW to provision │ = │ Already provided │ +│ K8sNodeReady, │ │ YOUR stub scripts │ │ StepSuccessCheck, │ +│ GpuCapacity... │ │ call your API │ │ FieldExistsCheck │ +└──────────────────┘ └──────────────────────┘ └────────────────────┘ ``` +**Two ways to use templates:** + +1. **Layered** (recommended for new providers): pair a template with a provider config + ```bash + isvctl test run -f templates/kaas.yaml -f my-provider/kaas.yaml + ``` + +2. **Self-contained** (existing approach): copy a template and add commands inline + ```bash + isvctl test run -f my-provider/kaas.yaml # has both commands + validations + ``` + **The contract is JSON.** Your scripts can be written in any language (Python, Bash, Go, etc.). They just need to print a JSON object to stdout with the required fields. ## Available Templates -| Template | Tests | Stubs | Reference Implementation | -|----------|-------|-------|--------------------------| -| `iam.yaml` | User create → verify credentials → delete | `stubs/iam/` (3 scripts) | `../stubs/aws/iam/` | -| `network.yaml` | VPC CRUD, subnets, isolation, security, connectivity, traffic | `stubs/network/` (8 scripts) | `../stubs/aws/network/` | -| `vm.yaml` | Launch GPU VM → list → reboot → NIM deploy → teardown | `stubs/vm/` (4 scripts) + `stubs/common/` (2) | `../stubs/aws/vm/` | -| `bm.yaml` | Launch bare-metal → describe → reboot → NIM → teardown → verify | `stubs/bm/` (5 scripts) + `stubs/common/` (2) | `../stubs/aws/bm/` | -| `kaas.yaml` | Provision K8s GPU cluster → validate nodes/GPU/workloads → teardown | `stubs/kaas/` (2 scripts) | `../stubs/aws/eks/` | -| `control-plane.yaml` | API health, access key lifecycle, tenant lifecycle | `stubs/control-plane/` (10 scripts) | `../stubs/aws/control-plane/` | -| `image-registry.yaml` | Image upload → VM launch → install config CRUD → BMaaS install → teardown | `stubs/image-registry/` (6 scripts) | `../stubs/aws/image-registry/` | +| Template | Validations | Provider Stubs | Reference Implementation | +|----------|-------------|----------------|--------------------------| +| `kaas.yaml` | K8s cluster health, GPU operator, GPU scheduling, workloads | `stubs/kaas/` (2 scripts) | `../aws/eks.yaml` | +| `control-plane.yaml` | API health, access key lifecycle, tenant lifecycle | `stubs/control-plane/` (10 scripts) | `../aws/control-plane.yaml` | +| `iam.yaml` | User create, verify credentials, delete | `stubs/iam/` (3 scripts) | `../aws/iam.yaml` | +| `network.yaml` | VPC CRUD, subnets, isolation, security, connectivity | `stubs/network/` (8 scripts) | `../aws/network.yaml` | +| `vm.yaml` | GPU VM lifecycle, SSH, NIM inference | `stubs/vm/` (4 scripts) + `stubs/common/` (2) | `../aws/vm.yaml` | +| `bm.yaml` | Bare-metal lifecycle, SSH, GPU, NIM | `stubs/bm/` (5 scripts) + `stubs/common/` (2) | `../aws/bm.yaml` | +| `image-registry.yaml` | Image upload, install config CRUD, BMaaS install | `stubs/image-registry/` (6 scripts) | `../aws/image-registry.yaml` | + +## Context Variables + +Templates use `{{ context.variable | default('value') }}` for provider-specific values. +Providers set these via the `context:` block in their YAML: + +```yaml +# my-provider/kaas.yaml +context: + node_count: "5" + total_gpus: "20" + gpu_per_node: "4" + gpu_operator_ns: "nvidia-gpu-operator" +``` -> **Note on Reference Implementations:** The `../stubs/aws/` paths in the "Reference Implementation" column point to NVIDIA's AWS example scripts that live _outside_ the `templates/` folder. These are optional examples provided as implementation guides — they are **not** copied when you duplicate `templates/` and are **not** required dependencies. The relative paths will not resolve once the templates folder is relocated. Refer to them in-place for inspiration, then implement your own scripts in the `stubs/` directories listed in the "Stubs" column. +Or override at runtime: +```bash +isvctl test run -f templates/kaas.yaml -f my-provider/kaas.yaml \ + --set context.node_count=10 +``` ## Quick Start +### Layered approach (new providers) + +```bash +# 1. Create a provider config with just commands + context +cat > isvctl/configs/my-isv/kaas.yaml << 'EOF' +version: "1.0" +commands: + kubernetes: + phases: ["setup", "test", "teardown"] + steps: + - name: provision_cluster + phase: setup + command: "../stubs/my-isv/kaas/setup.sh" + timeout: 1800 + - name: teardown_cluster + phase: teardown + command: "../stubs/my-isv/kaas/teardown.sh" + timeout: 1800 +context: + node_count: "3" + total_gpus: "12" +EOF + +# 2. Implement the stub scripts +vim isvctl/configs/stubs/my-isv/kaas/setup.sh + +# 3. Run with the template +isvctl test run -f isvctl/configs/templates/kaas.yaml -f isvctl/configs/my-isv/kaas.yaml +``` + +### Copy approach (existing workflow) + ```bash # 1. Copy the template folder cp -r isvctl/configs/templates/ isvctl/configs/my-isv/ diff --git a/isvctl/configs/templates/bm.yaml b/isvctl/configs/templates/bm.yaml index 3d336e8b..adf5c810 100644 --- a/isvctl/configs/templates/bm.yaml +++ b/isvctl/configs/templates/bm.yaml @@ -8,226 +8,39 @@ # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. -# Bare Metal (BMaaS) Validation - Template Configuration +# Bare Metal (BMaaS) Validation - Template # -# This is a PROVIDER-AGNOSTIC template for testing bare-metal GPU instance -# lifecycle: launch -> describe -> reboot -> NIM deploy -> teardown -> verify. -# Copy this file and the accompanying stubs/ scripts, then replace the -# script implementations with your platform's API calls. +# Provider-agnostic validations for bare-metal GPU instance lifecycle: +# launch -> describe -> reboot -> NIM deploy -> teardown -> verify. +# Pair with a provider config that supplies the commands (stubs). # -# Architecture: -# - Scripts: Platform-specific (YOU implement these) - do all BM work -# - Validations: Platform-agnostic (provided) - just check JSON output +# Usage: +# # Layered: template + provider +# isvctl test run -f isvctl/configs/templates/bm.yaml -f isvctl/configs/aws/bm.yaml # -# Contract: -# Each script MUST print a JSON object to stdout. The validations -# only inspect that JSON. As long as your scripts output the right -# fields, the validations pass regardless of which cloud you use. +# # With overrides +# isvctl test run -f ... --set context.expected_gpus=4 --set context.expected_os=rhel # -# Steps: -# 1. launch_instance (setup) - Provision bare-metal GPU instance -# 2. list_instances (test) - List instances, verify target exists -# 3. describe_instance (test) - Describe instance state + SSH info -# 4. reboot_instance (test) - Reboot instance, validate recovery -# 5. reinstall_instance (test) - Reinstall OS from stock image [skip: true by default] -# 6. deploy_nim (test) - Deploy NIM container via SSH -# 7. teardown_nim (teardown) - Stop NIM container -# 8. teardown (teardown) - Terminate instance -# 9. verify_teardown (teardown) - Confirm resources deleted -# -# Dev workflow (reuse existing instance): -# # Run 1: launch + test, keep instance alive -# BM_SKIP_TEARDOWN=true uv run isvctl test run -f isvctl/configs/my-isv/bm.yaml -# -# # Run 2+: reuse instance -# BM_INSTANCE_ID= BM_KEY_FILE=/path/to/key.pem \ -# BM_SKIP_TEARDOWN=true uv run isvctl test run -f isvctl/configs/my-isv/bm.yaml -# -# Quick start: -# 1. Copy this folder: cp -r templates/ my-isv/ -# 2. Edit the stubs: my-isv/stubs/bm/*.py -# 3. Run: uv run isvctl test run -f isvctl/configs/my-isv/bm.yaml -# -# See also: -# - AWS reference implementation: ../aws/bm.yaml + ../stubs/aws/bm/ -# - JSON schemas: isvctl/src/isvctl/config/output_schemas.py +# Context variables (set by provider or --set): +# context.expected_gpus — Number of GPUs in the instance (default: 8) +# context.expected_os — Expected OS name for SSH checks (default: ubuntu) +# context.max_reboot_uptime — Max uptime in seconds after reboot (default: 600) +# context.gpu_stress_runtime — GPU stress test duration in seconds (default: 30) +# context.gpu_stress_memory_gb — GPU stress test memory in GB (default: 16) +# context.training_steps — DDP training workload steps (default: 50) +# context.ping_target — Ethernet ping target (default: 8.8.8.8) +# context.nim_prompt — Prompt for NIM inference check (default: What is CUDA?) +# context.nim_max_tokens — Max tokens for NIM inference (default: 50) version: "1.0" -commands: - bm: - phases: ["setup", "test", "teardown"] - steps: - # ─── Step 1: Launch Bare-Metal Instance ──────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "bm", - # "instance_id": "", - # "public_ip": "", - # "key_file": "", - # "vpc_id": "", - # "instance_state": "running", - # "security_group_id": "", - # "key_name": "" - # } - - name: launch_instance - phase: setup - command: "python3 ./stubs/bm/launch_instance.py" - args: - - "--name" - - "isv-bm-test-gpu" - - "--instance-type" - - "{{instance_type}}" - - "--region" - - "{{region}}" - timeout: 1200 - - # ─── Step 2: List Instances ──────────────────────────────────── - # Reuses the VM list_instances stub (generic instance listing). - - name: list_instances - phase: test - command: "python3 ./stubs/vm/list_instances.py" - args: - - "--vpc-id" - - "{{steps.launch_instance.vpc_id}}" - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--region" - - "{{region}}" - timeout: 120 - - # ─── Step 3: Describe Instance ──────────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "bm", - # "instance_id": "...", - # "instance_state": "running", - # "public_ip": "...", - # "key_file": "..." - # } - - name: describe_instance - phase: test - command: "python3 ./stubs/bm/describe_instance.py" - args: - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--region" - - "{{region}}" - - "--key-file" - - "{{steps.launch_instance.key_file}}" - timeout: 120 - - # ─── Step 4: Reboot Instance ────────────────────────────────── - # BM-specific: longer waits for hardware POST/BIOS/OS boot. - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "bm", - # "instance_id": "...", - # "instance_state": "running", - # "public_ip": "...", - # "key_file": "...", - # "uptime_seconds": , - # "ssh_connectivity": true - # } - - name: reboot_instance - phase: test - command: "python3 ./stubs/bm/reboot_instance.py" - args: - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--region" - - "{{region}}" - - "--key-file" - - "{{steps.launch_instance.key_file}}" - - "--public-ip" - - "{{steps.launch_instance.public_ip}}" - timeout: 1800 - - # ─── Step 5: Reinstall Instance ───────────────────────────────── - # Reinstall the node from its configured stock OS. - # Skipped by default: can be very slow depending on platform. - # Set skip: false and implement stubs/bm/reinstall_instance.py to enable. - # See ../aws/bm.yaml + stubs/aws/bm/reinstall_instance.py for reference. - - name: reinstall_instance - phase: test - skip: true - command: "python3 ./stubs/bm/reinstall_instance.py" - args: - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--region" - - "{{region}}" - - "--key-file" - - "{{steps.launch_instance.key_file}}" - - "--public-ip" - - "{{steps.launch_instance.public_ip}}" - timeout: 3600 - - # ─── Step 6: Deploy NIM Container ───────────────────────────── - - name: deploy_nim - phase: test - command: "python3 ./stubs/common/deploy_nim.py" - args: - - "--host" - - "{{steps.launch_instance.public_ip}}" - - "--key-file" - - "{{steps.launch_instance.key_file}}" - timeout: 1800 - - # ─── Step 7: Teardown NIM ───────────────────────────────────── - - name: teardown_nim - phase: teardown - command: "python3 ./stubs/common/teardown_nim.py" - args: - - "--host" - - "{{steps.launch_instance.public_ip}}" - - "--key-file" - - "{{steps.launch_instance.key_file}}" - timeout: 120 - - # ─── Step 8: Teardown Instance ──────────────────────────────── - - name: teardown - phase: teardown - command: "python3 ./stubs/bm/teardown.py" - args: - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--region" - - "{{region}}" - - "--delete-key-pair" - - "--delete-security-group" - - "{{teardown_flag}}" - timeout: 1800 - - # ─── Step 9: Verify Teardown ────────────────────────────────── - # Post-teardown sanitization: verify instance terminated and - # resources deleted. - - name: verify_teardown - phase: teardown - command: "python3 ./stubs/bm/verify_terminated.py" - args: - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--region" - - "{{region}}" - - "--security-group-id" - - "{{steps.launch_instance.security_group_id}}" - - "--key-name" - - "{{steps.launch_instance.key_name}}" - timeout: 120 - tests: platform: bare_metal - cluster_name: "bm-validation" + cluster_name: "{{ context.cluster_name | default('bm-validation') }}" description: "Bare metal (BMaaS) GPU validation tests (template)" settings: - region: "us-west-2" - instance_type: "{{instance_type}}" # Supply your platform-equivalent bare-metal/GPU instance type - teardown_flag: "{{(env.BM_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" + show_skipped_tests: false # ─── Validations ───────────────────────────────────────────────── # These are PROVIDER-AGNOSTIC. They only check JSON field names/values. @@ -255,13 +68,13 @@ tests: checks: - SshConnectivityCheck: {} - SshOsCheck: - expected_os: "ubuntu" + expected_os: "{{ context.expected_os | default('ubuntu') }}" gpu: step: describe_instance checks: - SshGpuCheck: - expected_gpus: 8 + expected_gpus: "{{ context.expected_gpus | default('8') }}" host_os: step: describe_instance @@ -273,8 +86,8 @@ tests: step: describe_instance checks: - SshGpuStressCheck: - runtime: 30 - memory_gb: 16 + runtime: "{{ context.gpu_stress_runtime | default('30') }}" + memory_gb: "{{ context.gpu_stress_memory_gb | default('16') }}" # NCCL AllReduce - validates GPU-to-GPU communication (NVLink/NVSwitch) nccl: @@ -287,7 +100,7 @@ tests: step: describe_instance checks: - SshTrainingCheck: - steps: 50 + steps: "{{ context.training_steps | default('50') }}" # Network / interconnect checks nvlink: @@ -304,13 +117,13 @@ tests: step: describe_instance checks: - SshEthernetCheck: - ping_target: "8.8.8.8" + ping_target: "{{ context.ping_target | default('8.8.8.8') }}" reboot_checks: step: reboot_instance checks: - InstanceRebootCheck: - max_uptime: 600 + max_uptime: "{{ context.max_reboot_uptime | default('600') }}" reboot_state: step: reboot_instance @@ -323,13 +136,13 @@ tests: checks: - SshConnectivityCheck: {} - SshOsCheck: - expected_os: "ubuntu" + expected_os: "{{ context.expected_os | default('ubuntu') }}" reboot_gpu: step: reboot_instance checks: - SshGpuCheck: - expected_gpus: 8 + expected_gpus: "{{ context.expected_gpus | default('8') }}" reboot_host_os: step: reboot_instance @@ -348,7 +161,7 @@ tests: checks: - SshConnectivityCheck: {} - SshOsCheck: - expected_os: "ubuntu" + expected_os: "{{ context.expected_os | default('ubuntu') }}" reinstall_gpu: step: reinstall_instance @@ -369,8 +182,8 @@ tests: step: deploy_nim checks: - SshNimInferenceCheck: - prompt: "What is CUDA?" - max_tokens: 50 + prompt: "{{ context.nim_prompt | default('What is CUDA?') }}" + max_tokens: "{{ context.nim_max_tokens | default('50') }}" nim_teardown: step: teardown_nim diff --git a/isvctl/configs/templates/control-plane.yaml b/isvctl/configs/templates/control-plane.yaml index 2056a4e1..50ac7167 100644 --- a/isvctl/configs/templates/control-plane.yaml +++ b/isvctl/configs/templates/control-plane.yaml @@ -8,185 +8,32 @@ # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. -# Control Plane Validation - Template Configuration +# Control Plane Validation - Template # -# This is a PROVIDER-AGNOSTIC template for testing cloud control plane -# operations: API health, access key lifecycle, and tenant management. -# Copy this file and the accompanying stubs/ scripts, then replace the -# script implementations with your platform's API calls. +# Provider-agnostic validations for cloud control plane operations: +# API health, access key lifecycle, and tenant management. +# Pair with a provider config that supplies the commands (stubs). # -# Architecture: -# - Scripts: Platform-specific (YOU implement these) - CRUD operations -# - Validations: Platform-agnostic (provided) - just check JSON output +# Usage: +# # Layered: template + provider +# isvctl test run -f isvctl/configs/templates/control-plane.yaml -f isvctl/configs/aws/control-plane.yaml # -# Contract: -# Each script MUST print a JSON object to stdout. The validations -# only inspect that JSON. As long as your scripts output the right -# fields, the validations pass regardless of which cloud you use. +# # With overrides +# isvctl test run -f ... --set context.region=eu-west-1 # -# Test groups: -# 1. API Health - Authentication and service connectivity -# 2. Access Key Lifecycle - Create, authenticate, disable, verify rejection, delete -# 3. Tenant Lifecycle - Create, list, get info, delete -# -# Quick start: -# 1. Copy this folder: cp -r templates/ my-isv/ -# 2. Edit the stubs: my-isv/stubs/control-plane/*.py -# 3. Run: uv run isvctl test run -f isvctl/configs/my-isv/control-plane.yaml -# -# See also: -# - AWS reference implementation: ../aws/control-plane.yaml + ../stubs/aws/control-plane/ -# - JSON schemas: isvctl/src/isvctl/config/output_schemas.py +# Context variables (set by provider or --set): +# context.region — Cloud region for API calls (default: us-west-2) +# context.services — Comma-separated services to check (default: compute,storage,identity) version: "1.0" -commands: - control_plane: - phases: ["setup", "test", "teardown"] - steps: - # ═══════════════════════════════════════════════════════════════ - # SETUP: API Health Check - # ═══════════════════════════════════════════════════════════════ - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "control_plane", - # "account_id": "", - # "tests": {"auth": {"passed": true}, "service_name": {"passed": true}} - # } - - name: check_api - phase: setup - command: "python3 ./stubs/control-plane/check_api.py" - args: ["--region", "{{region}}", "--services", "{{services}}"] - timeout: 120 - - # ═══════════════════════════════════════════════════════════════ - # SETUP: Create Resources - # ═══════════════════════════════════════════════════════════════ - # Access key output: - # { - # "success": true, - # "platform": "control_plane", - # "username": "...", - # "access_key_id": "...", - # "secret_access_key": "..." - # } - - name: create_access_key - phase: setup - command: "python3 ./stubs/control-plane/create_access_key.py" - args: ["--region", "{{region}}"] - timeout: 60 - - # Tenant output: - # { - # "success": true, - # "platform": "control_plane", - # "tenant_name": "...", - # "tenant_id": "..." - # } - - name: create_tenant - phase: setup - command: "python3 ./stubs/control-plane/create_tenant.py" - args: ["--region", "{{region}}"] - timeout: 60 - - # ═══════════════════════════════════════════════════════════════ - # TEST: Access Key Lifecycle - # ═══════════════════════════════════════════════════════════════ - - name: test_access_key - phase: test - command: "python3 ./stubs/control-plane/test_access_key.py" - args: - - "--access-key-id" - - "{{steps.create_access_key.access_key_id}}" - - "--secret-access-key" - - "{{steps.create_access_key.secret_access_key}}" - - "--region" - - "{{region}}" - sensitive_args: ["--secret-access-key"] - timeout: 120 - - - name: disable_access_key - phase: test - command: "python3 ./stubs/control-plane/disable_access_key.py" - args: - - "--username" - - "{{steps.create_access_key.username}}" - - "--access-key-id" - - "{{steps.create_access_key.access_key_id}}" - - "--region" - - "{{region}}" - timeout: 60 - - - name: verify_key_rejected - phase: test - command: "python3 ./stubs/control-plane/verify_key_rejected.py" - args: - - "--access-key-id" - - "{{steps.create_access_key.access_key_id}}" - - "--secret-access-key" - - "{{steps.create_access_key.secret_access_key}}" - - "--region" - - "{{region}}" - sensitive_args: ["--secret-access-key"] - timeout: 120 - - # ═══════════════════════════════════════════════════════════════ - # TEST: Tenant Lifecycle - # ═══════════════════════════════════════════════════════════════ - - name: list_tenants - phase: test - command: "python3 ./stubs/control-plane/list_tenants.py" - args: - - "--region" - - "{{region}}" - - "--target-group" - - "{{steps.create_tenant.tenant_name}}" - timeout: 60 - - - name: get_tenant - phase: test - command: "python3 ./stubs/control-plane/get_tenant.py" - args: - - "--group-name" - - "{{steps.create_tenant.tenant_name}}" - - "--region" - - "{{region}}" - timeout: 60 - - # ═══════════════════════════════════════════════════════════════ - # TEARDOWN: Cleanup Resources - # ═══════════════════════════════════════════════════════════════ - - name: delete_access_key - phase: teardown - command: "python3 ./stubs/control-plane/delete_access_key.py" - args: - - "--username" - - "{{steps.create_access_key.username}}" - - "--access-key-id" - - "{{steps.create_access_key.access_key_id}}" - timeout: 60 - output_schema: teardown - - - name: delete_tenant - phase: teardown - command: "python3 ./stubs/control-plane/delete_tenant.py" - args: - - "--group-name" - - "{{steps.create_tenant.tenant_name}}" - - "--region" - - "{{region}}" - timeout: 60 - output_schema: teardown - tests: platform: control_plane - cluster_name: "control-plane-validation" + cluster_name: "{{ context.cluster_name | default('control-plane-validation') }}" description: "Control plane: API health, access key lifecycle, tenant lifecycle (template)" settings: - region: "us-west-2" - services: "compute,storage,identity" + show_skipped_tests: false # ─── Validations ───────────────────────────────────────────────── # These are PROVIDER-AGNOSTIC. They only check JSON field names/values. diff --git a/isvctl/configs/templates/iam.yaml b/isvctl/configs/templates/iam.yaml index 29927097..fa332d2b 100644 --- a/isvctl/configs/templates/iam.yaml +++ b/isvctl/configs/templates/iam.yaml @@ -8,117 +8,33 @@ # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. -# IAM User Lifecycle Validation - Template Configuration +# IAM User Lifecycle Validation - Template # -# This is a PROVIDER-AGNOSTIC template for testing user account -# create -> verify -> delete lifecycle. Copy this file and the -# accompanying stubs/ scripts, then replace the script implementations -# with your platform's API calls. +# Provider-agnostic validations for user account create -> verify -> delete +# lifecycle. Pair with a provider config that supplies the commands (stubs). # -# Architecture: -# - Scripts: Platform-specific (YOU implement these) - do all IAM work -# - Validations: Platform-agnostic (provided) - just check JSON output +# Usage: +# # Layered: template + provider +# isvctl test run -f isvctl/configs/templates/iam.yaml -f isvctl/configs/aws/iam.yaml # -# Contract: -# Each script MUST print a JSON object to stdout. The validations -# only inspect that JSON. As long as your scripts output the right -# fields, the validations pass regardless of which IAM system you use. +# # With overrides +# isvctl test run -f ... --set context.region=eu-west-1 # -# Steps: -# 1. create_user (setup) - Create a user account, output JSON -# 2. test_credentials (test) - Verify the credentials work, output JSON -# 3. teardown (teardown) - Delete the user, output JSON -# -# Quick start: -# 1. Copy this folder: cp -r templates/ my-isv/ -# 2. Edit the stubs: my-isv/stubs/iam/create_user.py (etc.) -# 3. Run: uv run isvctl test run -f isvctl/configs/my-isv/iam.yaml -# -# See also: -# - AWS reference implementation: ../aws/iam.yaml + ../stubs/aws/iam/ -# - JSON schemas: isvctl/src/isvctl/config/output_schemas.py +# Context variables (set by provider or --set): +# context.region — Cloud region for IAM calls (default: us-west-2) version: "1.0" -commands: - iam: - phases: ["setup", "test", "teardown"] - steps: - # ─── Step 1: Create User ─────────────────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "iam", - # "username": "", - # "user_id": "", - # "access_key_id": "", # if applicable - # "secret_access_key": "" # if applicable - # } - - name: create_user - phase: setup - command: "python3 ./stubs/iam/create_user.py" - args: - - "--username" - - "isv-test-user" - timeout: 60 - - # ─── Step 2: Test Credentials ────────────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "iam", - # "account_id": "", - # "tests": { - # "identity": {"passed": true}, - # "access": {"passed": true} - # } - # } - # - # The args below use Jinja2 templating to forward values from - # the create_user step output. Adjust arg names to match your script. - - name: test_credentials - phase: test - command: "python3 ./stubs/iam/test_credentials.py" - args: - - "--username" - - "{{steps.create_user.username}}" - - "--credential-id" - - "{{steps.create_user.access_key_id}}" - - "--credential-secret" - - "{{steps.create_user.secret_access_key}}" - sensitive_args: ["--credential-secret"] - timeout: 60 - - # ─── Step 3: Teardown (Delete User) ──────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "iam", - # "resources_deleted": ["user:"], - # "message": "User deleted successfully" - # } - - name: teardown - phase: teardown - command: "python3 ./stubs/iam/delete_user.py" - args: - - "--username" - - "{{steps.create_user.username}}" - - "{{teardown_flag}}" - timeout: 120 - tests: - cluster_name: "iam-validation" + cluster_name: "{{ context.cluster_name | default('iam-validation') }}" description: "IAM user lifecycle validation (template)" platform: iam settings: - # Override these for your environment - region: "us-west-2" - teardown_flag: "{{(env.IAM_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" + show_skipped_tests: false # ─── Validations ───────────────────────────────────────────────── # These are PROVIDER-AGNOSTIC. They only check JSON field names/values. - # You should NOT need to change these unless you add extra checks. validations: setup_checks: step: create_user diff --git a/isvctl/configs/templates/image-registry.yaml b/isvctl/configs/templates/image-registry.yaml index e8d93f77..ab1375fb 100644 --- a/isvctl/configs/templates/image-registry.yaml +++ b/isvctl/configs/templates/image-registry.yaml @@ -8,194 +8,32 @@ # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. -# Image Registry Validation - Template Configuration +# Image Registry Validation - Template # -# This is a PROVIDER-AGNOSTIC template for testing image registry lifecycle: -# - OS image management: upload, import, launch VM, teardown -# - OS install configuration CRUD (create, read, update, delete) -# - BMaaS provisioning from images and install configs +# Provider-agnostic validations for image registry lifecycle: +# OS image upload, VM launch from image, install config CRUD, +# and BMaaS provisioning from images and configs. +# Pair with a provider config that supplies the commands (stubs). # -# Architecture: -# - Scripts: Platform-specific (YOU implement these) - handle cloud operations -# - Validations: Platform-agnostic (provided) - just check JSON output -# - Schema: Uses generic field names (image_id, config_id, instance_id) +# Usage: +# # Layered: template + provider +# isvctl test run -f isvctl/configs/templates/image-registry.yaml -f isvctl/configs/aws/image-registry.yaml # -# Contract: -# Each script MUST print a JSON object to stdout. The validations -# only inspect that JSON. As long as your scripts output the right -# fields, the validations pass regardless of which cloud you use. +# # With overrides +# isvctl test run -f ... --set context.expected_os=rhel # -# Steps: -# 1. upload_image (setup) - Upload OS image to registry -# 2. launch_instance (test) - Launch VM from imported image -# 3. create_install_config (test) - CRUD an OS install configuration -# 4. install_image_bm (test) - Install OS image on bare-metal -# 5. install_config_bm (test) - Install OS config on bare-metal -# 6. teardown (teardown) - Clean up all resources -# -# Note: In the AWS reference implementation, the BM install/verify steps -# (install_image_bm, install_config_bm) are in ../aws/bm.yaml instead of -# here, as verify_image and verify_config steps within the BM lifecycle. -# -# Quick start: -# 1. Copy this folder: cp -r templates/ my-isv/ -# 2. Edit the stubs: my-isv/stubs/image-registry/*.py -# 3. Run: uv run isvctl test run -f isvctl/configs/my-isv/image-registry.yaml -# -# See also: -# - AWS reference implementation: ../aws/image-registry.yaml + ../stubs/aws/image-registry/ -# - JSON schemas: isvctl/src/isvctl/config/output_schemas.py +# Context variables (set by provider or --set): +# context.expected_os — Expected OS name for SSH checks (default: ubuntu) version: "1.0" -commands: - image_registry: - phases: ["setup", "test", "teardown"] - steps: - # ─── Step 1: Upload Image ────────────────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "image_registry", - # "image_id": "", - # "storage_bucket": "", - # "disk_ids": [""] - # } - - name: upload_image - phase: setup - command: "python3 ./stubs/image-registry/upload_image.py" - args: - - "--image-url" - - "{{image_url}}" - - "--image-format" - - "{{image_format}}" - - "--region" - - "{{region}}" - timeout: 3600 - - # ─── Step 2: Launch Instance from Image ──────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "image_registry", - # "instance_id": "", - # "public_ip": "", - # "key_path": "", - # "instance_state": "running", - # "key_name": "", - # "security_group_id": "", - # "instance_profile": "" - # } - - name: launch_instance - phase: test - command: "python3 ./stubs/image-registry/launch_instance.py" - args: - - "--image-id" - - "{{steps.upload_image.image_id}}" - - "--instance-type" - - "{{instance_type}}" - - "--region" - - "{{region}}" - timeout: 600 - - # ─── Step 3: CRUD OS Install Configuration ──────────────────── - # YOUR SCRIPT must create, read, update, and delete an OS - # install configuration (e.g., iPXE config, Carbide profile). - # { - # "success": true, - # "platform": "image_registry", - # "config_id": "", - # "config_name": "", - # "operations": { - # "create": {"passed": true}, - # "read": {"passed": true}, - # "update": {"passed": true}, - # "delete": {"passed": true} - # } - # } - - name: crud_install_config - phase: test - command: "python3 ./stubs/image-registry/crud_install_config.py" - args: - - "--region" - - "{{region}}" - timeout: 300 - - # ─── Step 4: Install OS Image on Bare-Metal ────────────────── - # YOUR SCRIPT must provision a BM node from the uploaded OS image. - # { - # "success": true, - # "platform": "image_registry", - # "instance_id": "", - # "image_id": "", - # "instance_state": "running" - # } - - name: install_image_bm - phase: test - command: "python3 ./stubs/image-registry/install_image_bm.py" - args: - - "--image-id" - - "{{steps.upload_image.image_id}}" - - "--region" - - "{{region}}" - timeout: 1800 - - # ─── Step 5: Install OS Config on Bare-Metal ───────────────── - # YOUR SCRIPT must provision a BM node using an install config. - # { - # "success": true, - # "platform": "image_registry", - # "instance_id": "", - # "config_id": "", - # "instance_state": "running" - # } - - name: install_config_bm - phase: test - command: "python3 ./stubs/image-registry/install_config_bm.py" - args: - - "--config-id" - - "{{steps.crud_install_config.config_id}}" - - "--region" - - "{{region}}" - timeout: 1800 - - # ─── Step 6: Teardown ────────────────────────────────────────── - # YOUR SCRIPT must clean up all resources: instances, images, - # configs, storage, keys, security groups. - - name: teardown - phase: teardown - command: "python3 ./stubs/image-registry/teardown.py" - args: - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--image-id" - - "{{steps.upload_image.image_id}}" - - "--disk-ids" - - "{{steps.upload_image.disk_ids | join(',')}}" - - "--bucket-name" - - "{{steps.upload_image.storage_bucket}}" - - "--key-name" - - "{{steps.launch_instance.key_name}}" - - "--security-group-id" - - "{{steps.launch_instance.security_group_id}}" - - "--instance-profile" - - "{{steps.launch_instance.instance_profile}}" - - "--region" - - "{{region}}" - - "{{teardown_flag}}" - timeout: 1800 - tests: - cluster_name: "image-registry-validation" + cluster_name: "{{ context.cluster_name | default('image-registry-validation') }}" description: "Image registry validation tests (template)" platform: image_registry settings: - region: "us-west-2" - image_url: "https://cloud-images.ubuntu.com/releases/noble/release/ubuntu-24.04-server-cloudimg-amd64.vmdk" - image_format: "vmdk" - instance_type: "g4dn.xlarge" - teardown_flag: "{{(env.IR_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" + show_skipped_tests: false # ─── Validations ───────────────────────────────────────────────── # These are PROVIDER-AGNOSTIC. They only check JSON field names/values. @@ -223,7 +61,7 @@ tests: checks: - SshConnectivityCheck: {} - SshOsCheck: - expected_os: "ubuntu" + expected_os: "{{ context.expected_os | default('ubuntu') }}" # --- OS Install Config CRUD --- install_config_crud: diff --git a/isvctl/configs/templates/kaas.yaml b/isvctl/configs/templates/kaas.yaml index a0d89caf..7cd5b0ae 100644 --- a/isvctl/configs/templates/kaas.yaml +++ b/isvctl/configs/templates/kaas.yaml @@ -8,104 +8,61 @@ # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. -# Kubernetes-as-a-Service (KaaS) Cluster Validation - Template Configuration +# Kubernetes-as-a-Service (KaaS) Cluster Validation - Template # -# This is a PROVIDER-AGNOSTIC template for testing GPU-enabled Kubernetes -# cluster lifecycle: provision -> validate -> workloads -> teardown. -# Copy this file and the accompanying stubs/ scripts, then replace the -# script implementations with your platform's provisioning logic. +# Provider-agnostic validations for GPU-enabled Kubernetes clusters. +# Pair with a provider config that supplies the commands (stubs). # -# Architecture: -# - Scripts: Platform-specific (YOU implement these) - provision/teardown cluster -# - Validations: Platform-agnostic (provided) - Kubernetes API checks +# Usage: +# # Layered: template + provider +# isvctl test run -f isvctl/configs/templates/kaas.yaml -f isvctl/configs/aws/eks.yaml # -# Contract: -# Setup script must configure kubectl access (e.g., write kubeconfig) -# and output a JSON object to stdout. Validations use kubectl directly. +# # With overrides +# isvctl test run -f ... --set context.node_count=5 --set context.total_gpus=4 # -# Steps: -# 1. provision_cluster (setup) - Provision K8s cluster, configure kubectl -# 2. teardown_cluster (teardown) - Destroy cluster and resources -# -# Quick start: -# 1. Copy this folder: cp -r templates/ my-isv/ -# 2. Edit the stubs: my-isv/stubs/kaas/setup.sh, teardown.sh -# 3. Run: uv run isvctl test run -f isvctl/configs/my-isv/kaas.yaml -# -# See also: -# - AWS reference implementation: ../aws/eks.yaml + ../stubs/aws/eks/ -# - Terraform example: ../stubs/aws/eks/terraform/ +# Context variables (set by provider or --set): +# context.node_count — Expected node count (default: 3) +# context.gpu_per_node — GPUs per node (default: 1) +# context.total_gpus — Total GPUs in cluster (default: 3) +# context.gpu_node_count — Number of GPU nodes (default: 1) +# context.runtime_class — RuntimeClass for GPU pods (default: nvidia) +# context.driver_version — Expected NVIDIA driver version (default: 570.195.03) +# context.gpu_operator_ns — GPU Operator namespace (default: gpu-operator) +# context.min_nccl_bw — Min NCCL bus bandwidth in Gbps (default: 100) version: "1.0" -commands: - kubernetes: - phases: ["setup", "test", "teardown"] - steps: - # ─── Setup: Provision Cluster ────────────────────────────────── - # YOUR SCRIPT must: - # 1. Provision a Kubernetes cluster with GPU nodes - # 2. Configure kubectl (write kubeconfig) - # 3. Print JSON to stdout: - # { - # "success": true, - # "platform": "kubernetes", - # "cluster_name": "", - # "cluster_endpoint": "", - # "node_count": , - # "gpu_node_count": - # } - - name: provision_cluster - phase: setup - command: "./stubs/kaas/setup.sh" - timeout: 1800 - env: - CLUSTER_REGION: "us-west-2" - - # ─── Teardown: Destroy Cluster ──────────────────────────────── - # YOUR SCRIPT must destroy all cluster resources. - - name: teardown_cluster - phase: teardown - command: "./stubs/kaas/teardown.sh" - timeout: 1800 - env: - # Override with TEARDOWN_ENABLED=false to preserve resources for debugging - TEARDOWN_ENABLED: "true" - tests: - cluster_name: "{{inventory.cluster_name}}" - description: "Kubernetes GPU cluster validation (template)" + cluster_name: "{{ context.cluster_name | default(inventory.cluster_name | default('')) }}" + description: "Kubernetes GPU cluster validation" platform: kubernetes settings: show_skipped_tests: false - # ─── Validations ───────────────────────────────────────────────── - # These use kubectl (configured by setup script) to check cluster state. - # Adjust counts and settings to match your cluster configuration. validations: kubernetes: - K8sNodeCountCheck: - count: "3" + count: "{{ context.node_count | default('3') }}" - K8sNodeReadyCheck: require_all_ready: true - K8sNvidiaSmiCheck: - runtime_class: "nvidia" + runtime_class: "{{ context.runtime_class | default('nvidia') }}" - K8sDriverVersionCheck: - driver_version: "570.195.03" - runtime_class: "nvidia" + driver_version: "{{ context.driver_version | default('570.195.03') }}" + runtime_class: "{{ context.runtime_class | default('nvidia') }}" - K8sGpuPodAccessCheck: gpu_count: 1 - total_gpu_count: 1 - runtime_class: "nvidia" + total_gpu_count: "{{ context.gpu_node_count | default('1') }}" + runtime_class: "{{ context.runtime_class | default('nvidia') }}" - K8sGpuCapacityCheck: resource_name: "nvidia.com/gpu" - expected_per_node: 1 - expected_total: 3 + expected_per_node: "{{ context.gpu_per_node | default('1') }}" + expected_total: "{{ context.total_gpus | default('3') }}" - K8sGpuOperatorNamespaceCheck: - namespace: "gpu-operator" + namespace: "{{ context.gpu_operator_ns | default('gpu-operator') }}" - K8sGpuOperatorPodsCheck: - namespace: "gpu-operator" + namespace: "{{ context.gpu_operator_ns | default('gpu-operator') }}" - K8sGpuLabelsCheck: label_selector: "nvidia.com/gpu.present=true" - K8sPodHealthCheck: @@ -116,7 +73,7 @@ tests: k8s_workloads: - K8sNcclWorkload: - min_bus_bw_gbps: 100 + min_bus_bw_gbps: "{{ context.min_nccl_bw | default('100') }}" - K8sGpuStressWorkload: memory_gb: 1 runtime: 30 diff --git a/isvctl/configs/templates/network.yaml b/isvctl/configs/templates/network.yaml index 4fdabc83..6e3a7f2b 100644 --- a/isvctl/configs/templates/network.yaml +++ b/isvctl/configs/templates/network.yaml @@ -8,183 +8,34 @@ # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. -# Network Validation - Template Configuration +# Network Validation - Template # -# This is a PROVIDER-AGNOSTIC template for comprehensive network testing. -# Copy this file and the accompanying stubs/ scripts, then replace the -# script implementations with your platform's API calls. +# Provider-agnostic validations for comprehensive network testing: +# VPC CRUD, subnet configuration, VPC isolation, security blocking, +# connectivity, and traffic validation. +# Pair with a provider config that supplies the commands (stubs). # -# Architecture: -# - Scripts: Platform-specific (YOU implement these) - do all network work -# - Validations: Platform-agnostic (provided) - just check JSON output +# Usage: +# # Layered: template + provider +# isvctl test run -f isvctl/configs/templates/network.yaml -f isvctl/configs/aws/network.yaml # -# Contract: -# Each script MUST print a JSON object to stdout. The validations -# only inspect that JSON. As long as your scripts output the right -# fields, the validations pass regardless of which cloud you use. +# # With overrides +# isvctl test run -f ... --set context.region=eu-west-1 --set context.min_subnets=3 # -# Test suites: -# 1. VPC CRUD - Create, Read, Update, Delete lifecycle -# 2. Subnet Configuration - Multi-AZ subnet distribution -# 3. VPC Isolation - Security boundaries between VPCs -# 4. Security Blocking - Firewall / ACL blocking rules (negative tests) -# 5. Connectivity - Instance network assignment -# 6. Traffic Validation - Real ping tests between instances -# -# Quick start: -# 1. Copy this folder: cp -r templates/ my-isv/ -# 2. Edit the stubs: my-isv/stubs/network/*.py -# 3. Run: uv run isvctl test run -f isvctl/configs/my-isv/network.yaml -# -# See also: -# - AWS reference implementation: ../aws/network.yaml + ../stubs/aws/network/ -# - JSON schemas: isvctl/src/isvctl/config/output_schemas.py +# Context variables (set by provider or --set): +# context.region — Cloud region for network calls (default: us-west-2) +# context.min_subnets — Minimum subnets for setup check (default: 2) +# context.subnet_count — Expected subnet count for subnet config test (default: 4) version: "1.0" -commands: - network: - phases: ["setup", "test", "teardown"] - steps: - # ─── Setup: Create Shared VPC ────────────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "network", - # "network_id": "", - # "subnets": [{"subnet_id": "..."}], - # "security_group_id": "" - # } - - name: create_network - phase: setup - command: "python3 ./stubs/network/create_vpc.py" - args: - - "--name" - - "isv-shared-vpc" - - "--region" - - "{{region}}" - - "--cidr" - - "10.0.0.0/16" - timeout: 300 - - # ─── Test 1: VPC CRUD Operations ─────────────────────────────── - # Self-contained test. Creates its own VPC, tests CRUD, cleans up. - # YOUR SCRIPT must output JSON matching the vpc_crud schema. - - name: vpc_crud - phase: test - command: "python3 ./stubs/network/vpc_crud_test.py" - args: - - "--region" - - "{{region}}" - - "--cidr" - - "10.99.0.0/16" - timeout: 120 - output_schema: vpc_crud - - # ─── Test 2: Subnet Configuration ────────────────────────────── - # Self-contained test. Creates VPC with multiple subnets, verifies. - # YOUR SCRIPT must output JSON matching the subnet_config schema. - - name: subnet_config - phase: test - command: "python3 ./stubs/network/subnet_test.py" - args: - - "--region" - - "{{region}}" - - "--cidr" - - "10.98.0.0/16" - - "--subnet-count" - - "4" - timeout: 120 - output_schema: subnet_config - - # ─── Test 3: VPC Isolation ───────────────────────────────────── - # Self-contained test. Creates two VPCs, verifies no cross-access. - # YOUR SCRIPT must output JSON matching the vpc_isolation schema. - - name: vpc_isolation - phase: test - command: "python3 ./stubs/network/isolation_test.py" - args: - - "--region" - - "{{region}}" - - "--cidr-a" - - "10.97.0.0/16" - - "--cidr-b" - - "10.96.0.0/16" - timeout: 120 - output_schema: vpc_isolation - - # ─── Test 4: Security Blocking ───────────────────────────────── - # Self-contained test. Tests firewall/ACL blocking rules. - # YOUR SCRIPT must output JSON matching the security_blocking schema. - - name: security_blocking - phase: test - command: "python3 ./stubs/network/security_test.py" - args: - - "--region" - - "{{region}}" - - "--cidr" - - "10.94.0.0/16" - timeout: 120 - output_schema: security_blocking - - # ─── Test 5: Connectivity ────────────────────────────────────── - # Uses the shared VPC from create_network step. - # YOUR SCRIPT must output JSON matching the connectivity_result schema. - - name: connectivity_test - phase: test - command: "python3 ./stubs/network/test_connectivity.py" - args: - - "--vpc-id" - - "{{steps.create_network.network_id}}" - - "--subnet-ids" - - "{{steps.create_network.subnets | map(attribute='subnet_id') | join(',')}}" - - "--sg-id" - - "{{steps.create_network.security_group_id}}" - - "--region" - - "{{region}}" - timeout: 600 - output_schema: connectivity_result - - # ─── Test 6: Traffic Validation ──────────────────────────────── - # Self-contained test. Real traffic tests between instances. - # YOUR SCRIPT must output JSON matching the traffic_flow schema. - - name: traffic_validation - phase: test - command: "python3 ./stubs/network/traffic_test.py" - args: - - "--region" - - "{{region}}" - - "--cidr" - - "10.93.0.0/16" - timeout: 900 - output_schema: traffic_flow - - # ─── Teardown: Clean up shared VPC ───────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "network", - # "resources_deleted": [...], - # "message": "..." - # } - - name: teardown - phase: teardown - command: "python3 ./stubs/network/teardown.py" - args: - - "--vpc-id" - - "{{steps.create_network.network_id}}" - - "--region" - - "{{region}}" - timeout: 300 - output_schema: teardown - tests: - cluster_name: "network-validation" + cluster_name: "{{ context.cluster_name | default('network-validation') }}" description: "Network validation tests (template)" platform: network settings: - region: "us-west-2" + show_skipped_tests: false # ─── Validations ───────────────────────────────────────────────── # These are PROVIDER-AGNOSTIC. They only check JSON field names/values. @@ -194,7 +45,7 @@ tests: checks: - NetworkProvisionedCheck: require_subnets: true - min_subnets: 2 + min_subnets: "{{ context.min_subnets | default('2') }}" network: checks: @@ -202,7 +53,7 @@ tests: step: vpc_crud - SubnetConfigCheck: step: subnet_config - min_subnets: 4 + min_subnets: "{{ context.subnet_count | default('4') }}" require_multi_az: true - VpcIsolationCheck: step: vpc_isolation diff --git a/isvctl/configs/templates/vm.yaml b/isvctl/configs/templates/vm.yaml index 31fff2c7..3ef3ff0b 100644 --- a/isvctl/configs/templates/vm.yaml +++ b/isvctl/configs/templates/vm.yaml @@ -8,168 +8,35 @@ # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. -# VM (Virtual Machine) Validation - Template Configuration +# VM (Virtual Machine) Validation - Template # -# This is a PROVIDER-AGNOSTIC template for testing GPU virtual machine -# lifecycle: launch -> verify -> reboot -> NIM deploy -> teardown. -# Copy this file and the accompanying stubs/ scripts, then replace the -# script implementations with your platform's API calls. +# Provider-agnostic validations for GPU virtual machine lifecycle: +# launch -> verify -> reboot -> NIM deploy -> teardown. +# Pair with a provider config that supplies the commands (stubs). # -# Architecture: -# - Scripts: Platform-specific (YOU implement these) - do all VM work -# - Validations: Platform-agnostic (provided) - just check JSON output +# Usage: +# # Layered: template + provider +# isvctl test run -f isvctl/configs/templates/vm.yaml -f isvctl/configs/aws/vm.yaml # -# Contract: -# Each script MUST print a JSON object to stdout. The validations -# only inspect that JSON. As long as your scripts output the right -# fields, the validations pass regardless of which cloud you use. +# # With overrides +# isvctl test run -f ... --set context.expected_gpus=4 --set context.expected_os=rhel # -# Steps: -# 1. launch_instance (setup) - Provision GPU VM, output IP/key -# 2. list_instances (test) - List instances, verify target exists -# 3. reboot_instance (test) - Reboot VM, validate recovery -# 4. deploy_nim (test) - Deploy NIM container via SSH -# 5. teardown_nim (teardown) - Stop NIM container via SSH -# 6. teardown (teardown) - Terminate VM and cleanup -# -# Quick start: -# 1. Copy this folder: cp -r templates/ my-isv/ -# 2. Edit the stubs: my-isv/stubs/vm/*.py -# 3. Run: uv run isvctl test run -f isvctl/configs/my-isv/vm.yaml -# -# See also: -# - AWS reference implementation: ../aws/vm.yaml + ../stubs/aws/vm/ -# - JSON schemas: isvctl/src/isvctl/config/output_schemas.py +# Context variables (set by provider or --set): +# context.expected_gpus — Number of GPUs in the VM (default: 1) +# context.expected_os — Expected OS name for SSH checks (default: ubuntu) +# context.max_reboot_uptime — Max uptime in seconds after reboot (default: 600) +# context.nim_prompt — Prompt for NIM inference check (default: What is CUDA?) +# context.nim_max_tokens — Max tokens for NIM inference (default: 50) version: "1.0" -commands: - vm: - phases: ["setup", "test", "teardown"] - steps: - # ─── Step 1: Launch Instance ─────────────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "vm", - # "instance_id": "", - # "public_ip": "", - # "key_file": "", - # "vpc_id": "", - # "instance_state": "running", - # "security_group_id": "", - # "key_name": "" - # } - - name: launch_instance - phase: setup - command: "python3 ./stubs/vm/launch_instance.py" - args: - - "--name" - - "isv-test-gpu" - - "--instance-type" - - "{{instance_type}}" - - "--region" - - "{{region}}" - timeout: 600 - - # ─── Step 2: List Instances ──────────────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "vm", - # "instances": [{"instance_id": "...", "state": "running"}], - # "total_count": 1 - # } - - name: list_instances - phase: test - command: "python3 ./stubs/vm/list_instances.py" - args: - - "--vpc-id" - - "{{steps.launch_instance.vpc_id}}" - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--region" - - "{{region}}" - timeout: 120 - - # ─── Step 3: Reboot Instance ────────────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "vm", - # "instance_id": "...", - # "instance_state": "running", - # "public_ip": "...", - # "key_file": "...", - # "uptime_seconds": , - # "ssh_connectivity": true - # } - - name: reboot_instance - phase: test - command: "python3 ./stubs/vm/reboot_instance.py" - args: - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--region" - - "{{region}}" - - "--key-file" - - "{{steps.launch_instance.key_file}}" - - "--public-ip" - - "{{steps.launch_instance.public_ip}}" - timeout: 600 - - # ─── Step 4: Deploy NIM Container ───────────────────────────── - # Shared script - deploys NIM inference container via SSH. - # See stubs/common/deploy_nim.py for the JSON contract. - - name: deploy_nim - phase: test - command: "python3 ./stubs/common/deploy_nim.py" - args: - - "--host" - - "{{steps.launch_instance.public_ip}}" - - "--key-file" - - "{{steps.launch_instance.key_file}}" - timeout: 1800 - - # ─── Step 5: Teardown NIM ───────────────────────────────────── - - name: teardown_nim - phase: teardown - command: "python3 ./stubs/common/teardown_nim.py" - args: - - "--host" - - "{{steps.launch_instance.public_ip}}" - - "--key-file" - - "{{steps.launch_instance.key_file}}" - timeout: 120 - - # ─── Step 6: Teardown Instance ──────────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "vm", - # "resources_deleted": [...], - # "message": "..." - # } - - name: teardown - phase: teardown - command: "python3 ./stubs/vm/teardown.py" - args: - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--region" - - "{{region}}" - - "--delete-key-pair" - - "--delete-security-group" - timeout: 600 - tests: platform: vm - cluster_name: "vm-validation" + cluster_name: "{{ context.cluster_name | default('vm-validation') }}" description: "VM (virtual machine) GPU validation tests (template)" settings: - region: "us-west-2" - instance_type: "g5.xlarge" + show_skipped_tests: false # ─── Validations ───────────────────────────────────────────────── # These are PROVIDER-AGNOSTIC. They only check JSON field names/values. @@ -191,27 +58,27 @@ tests: checks: - SshConnectivityCheck: {} - SshOsCheck: - expected_os: "ubuntu" + expected_os: "{{ context.expected_os | default('ubuntu') }}" gpu: step: launch_instance checks: - SshGpuCheck: - expected_gpus: 1 + expected_gpus: "{{ context.expected_gpus | default('1') }}" host_os: step: launch_instance checks: - SshVcpuPinningCheck: {} - SshPciBusCheck: - expected_gpus: 1 + expected_gpus: "{{ context.expected_gpus | default('1') }}" - SshHostSoftwareCheck: {} reboot_checks: step: reboot_instance checks: - InstanceRebootCheck: - max_uptime: 600 + max_uptime: "{{ context.max_reboot_uptime | default('600') }}" reboot_state: step: reboot_instance @@ -224,20 +91,20 @@ tests: checks: - SshConnectivityCheck: {} - SshOsCheck: - expected_os: "ubuntu" + expected_os: "{{ context.expected_os | default('ubuntu') }}" reboot_gpu: step: reboot_instance checks: - SshGpuCheck: - expected_gpus: 1 + expected_gpus: "{{ context.expected_gpus | default('1') }}" reboot_host_os: step: reboot_instance checks: - SshVcpuPinningCheck: {} - SshPciBusCheck: - expected_gpus: 1 + expected_gpus: "{{ context.expected_gpus | default('1') }}" - SshHostSoftwareCheck: {} nim_health: @@ -254,8 +121,8 @@ tests: step: deploy_nim checks: - SshNimInferenceCheck: - prompt: "What is CUDA?" - max_tokens: 50 + prompt: "{{ context.nim_prompt | default('What is CUDA?') }}" + max_tokens: "{{ context.nim_max_tokens | default('50') }}" nim_teardown: step: teardown_nim diff --git a/isvctl/src/isvctl/config/merger.py b/isvctl/src/isvctl/config/merger.py index d4a200dc..016f1479 100644 --- a/isvctl/src/isvctl/config/merger.py +++ b/isvctl/src/isvctl/config/merger.py @@ -17,17 +17,110 @@ """ import copy +import logging from pathlib import Path from typing import Any import yaml +logger = logging.getLogger(__name__) + + +_MERGE_MARKER = "__merge__" +_REMOVE_SENTINEL = "__remove__" + + +def _is_mergeable_list(lst: list[Any]) -> bool: + """Check if a list can be strategically merged. + + Returns True if the list is non-empty and every item is a dict with exactly + one key. This pattern matches config lists like checks where each item is + ``{"CheckName": {params}}``. + """ + if not lst: + return False + return all(isinstance(item, dict) and len(item) == 1 for item in lst) + + +def _has_merge_marker(lst: list[Any]) -> bool: + """Check if a list contains the ``{__merge__: true}`` opt-in marker. + + Only the exact shape ``{"__merge__": True}`` triggers strategic merge. + """ + return any( + isinstance(item, dict) and len(item) == 1 + and item.get(_MERGE_MARKER) is True + for item in lst + ) + + +def _strip_merge_marker(lst: list[Any]) -> list[Any]: + """Return a copy of the list with the ``__merge__`` marker removed.""" + return [ + item for item in lst + if not (isinstance(item, dict) and len(item) == 1 and _MERGE_MARKER in item) + ] + + +def _merge_single_key_dict_lists( + base_list: list[dict[str, Any]], override_list: list[dict[str, Any]] +) -> list[dict[str, Any]]: + """Merge two lists of single-key dicts by matching on the dict key. + + - Matching key: deep-merge the params (override wins on conflicts) + - New key in override: append to end + - Key not in override: keep unchanged + - Value set to ``"__remove__"``: delete the item + + Preserves base list order; new items from override appended at end. + """ + # Build an index of override items keyed by their single key + override_by_key: dict[str, Any] = {} + override_order: list[str] = [] + for item in override_list: + key = next(iter(item)) + override_by_key[key] = item[key] + override_order.append(key) + + seen_keys: set[str] = set() + result: list[dict[str, Any]] = [] + + # Walk base list, merging or removing as needed + for item in base_list: + key = next(iter(item)) + seen_keys.add(key) + + if key in override_by_key: + override_value = override_by_key[key] + if override_value == _REMOVE_SENTINEL: + continue # Drop this item + base_value = item[key] + if isinstance(base_value, dict) and isinstance(override_value, dict): + merged = deep_merge(base_value, override_value) + else: + merged = copy.deepcopy(override_value) + result.append({key: merged}) + else: + result.append(copy.deepcopy(item)) + + # Append new items from override that weren't in base (deduplicated) + for key in override_order: + if key not in seen_keys: + seen_keys.add(key) + if override_by_key[key] == _REMOVE_SENTINEL: + continue # Removing nonexistent key is a no-op + result.append({key: copy.deepcopy(override_by_key[key])}) + + return result + def deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: """Deep merge two dictionaries. Values from `override` take precedence. Nested dicts are merged recursively. - Lists are replaced entirely (not concatenated). + Lists are replaced entirely by default. When the override list contains a + ``{__merge__: true}`` marker and both lists are single-key dict lists, they + are merged by matching on the dict key instead. Args: base: Base dictionary @@ -42,9 +135,34 @@ def deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any] if key in result and isinstance(result[key], dict) and isinstance(value, dict): # Recursively merge nested dicts result[key] = deep_merge(result[key], value) + elif ( + key in result + and isinstance(result[key], list) + and isinstance(value, list) + and _has_merge_marker(value) + ): + # Opt-in strategic merge for lists of single-key dicts + override_items = _strip_merge_marker(value) + if _is_mergeable_list(result[key]) and _is_mergeable_list(override_items): + override_keys = [next(iter(item)) for item in override_items] + logger.debug( + "Strategic merge on '%s': %d base items, %d override items %s", + key, len(result[key]), len(override_items), override_keys, + ) + result[key] = _merge_single_key_dict_lists(result[key], override_items) + else: + logger.debug( + "Merge marker on '%s' but lists not mergeable, replacing", key, + ) + result[key] = copy.deepcopy(override_items) else: # Override with new value (including None) - result[key] = copy.deepcopy(value) + # Strip any __merge__ markers from lists to avoid leaking into runtime config + if isinstance(value, list): + stripped = _strip_merge_marker(value) + result[key] = copy.deepcopy(stripped) + else: + result[key] = copy.deepcopy(value) return result diff --git a/isvctl/tests/test_merger.py b/isvctl/tests/test_merger.py index 231c87aa..c0bddd58 100644 --- a/isvctl/tests/test_merger.py +++ b/isvctl/tests/test_merger.py @@ -10,12 +10,22 @@ """Tests for YAML merging functionality.""" +import copy from pathlib import Path from typing import Any import pytest -from isvctl.config.merger import apply_set_value, deep_merge, merge_yaml_files, parse_set_value +from isvctl.config.merger import ( + _has_merge_marker, + _is_mergeable_list, + _merge_single_key_dict_lists, + _strip_merge_marker, + apply_set_value, + deep_merge, + merge_yaml_files, + parse_set_value, +) class TestDeepMerge: @@ -42,6 +52,22 @@ def test_list_replacement(self) -> None: result = deep_merge(base, override) assert result == {"items": [4, 5]} + def test_remove_string_is_literal_at_dict_level(self) -> None: + """__remove__ at dict level is treated as a literal string value.""" + base = {"a": 1, "b": 2, "c": 3} + override = {"b": "__remove__"} + result = deep_merge(base, override) + # __remove__ only works inside strategic-merge lists, not at dict level + assert result == {"a": 1, "b": "__remove__", "c": 3} + + def test_remove_only_in_strategic_merge_lists(self) -> None: + """__remove__ deletes items only within strategic-merge lists.""" + base = {"checks": [{"A": {"p": 1}}, {"B": {"p": 2}}]} + override = {"checks": [{"__merge__": True}, {"B": "__remove__"}]} + result = deep_merge(base, override) + # B is removed from the list, A is kept + assert result == {"checks": [{"A": {"p": 1}}]} + def test_original_not_modified(self) -> None: """Test that original dicts are not modified.""" base = {"a": {"b": 1}} @@ -183,3 +209,246 @@ def test_empty_file_ignored(self, tmp_path: Path) -> None: result = merge_yaml_files([str(file1), str(file2)]) assert result == {"a": 1} + + +class TestStrategicMerge: + """Tests for opt-in strategic merge of single-key dict lists. + + Strategic merge is triggered by including ``{__merge__: true}`` in the + override list. Without the marker, lists are replaced as before. + """ + + # -- marker helpers ------------------------------------------------------- + + def test_has_merge_marker(self) -> None: + """_has_merge_marker detects the opt-in marker.""" + assert _has_merge_marker([{"__merge__": True}]) is True + assert _has_merge_marker([{"__merge__": True}, {"A": 1}]) is True + assert _has_merge_marker([{"A": 1}]) is False + assert _has_merge_marker([]) is False + + def test_strip_merge_marker(self) -> None: + """_strip_merge_marker removes only the marker item.""" + lst = [{"__merge__": True}, {"A": 1}, {"B": 2}] + assert _strip_merge_marker(lst) == [{"A": 1}, {"B": 2}] + + def test_is_mergeable_list_helper(self) -> None: + """Direct tests for _is_mergeable_list.""" + assert _is_mergeable_list([{"A": 1}]) is True + assert _is_mergeable_list([{"A": 1}, {"B": 2}]) is True + assert _is_mergeable_list([]) is False + assert _is_mergeable_list([1, 2]) is False + assert _is_mergeable_list(["a", "b"]) is False + assert _is_mergeable_list([{"A": 1, "B": 2}]) is False + assert _is_mergeable_list([{"A": 1}, "b"]) is False + + # -- opt-in behavior via deep_merge --------------------------------------- + + def test_no_marker_replaces_list(self) -> None: + """Without __merge__, single-key dict lists are replaced (backward compat).""" + base = {"checks": [{"A": {"p": 1}}, {"B": {"p": 2}}]} + override = {"checks": [{"A": {"p": 99}}]} + result = deep_merge(base, override) + # B is gone — full replacement + assert result == {"checks": [{"A": {"p": 99}}]} + + def test_merge_matching_check_params(self) -> None: + """Core case: merge params of a matching check.""" + base = {"checks": [{"CheckA": {"param1": 1, "param2": 2}}]} + override = {"checks": [{"__merge__": True}, {"CheckA": {"param2": 99}}]} + result = deep_merge(base, override) + assert result == {"checks": [{"CheckA": {"param1": 1, "param2": 99}}]} + + def test_append_new_check(self) -> None: + """New check in override is appended to end.""" + base = {"checks": [{"CheckA": {"p": 1}}]} + override = {"checks": [{"__merge__": True}, {"CheckB": {"p": 2}}]} + result = deep_merge(base, override) + assert result == {"checks": [{"CheckA": {"p": 1}}, {"CheckB": {"p": 2}}]} + + def test_remove_check_via_sentinel(self) -> None: + """Check removed when value is '__remove__'.""" + base = {"checks": [{"CheckA": {"p": 1}}, {"CheckB": {"p": 2}}]} + override = {"checks": [{"__merge__": True}, {"CheckA": "__remove__"}]} + result = deep_merge(base, override) + assert result == {"checks": [{"CheckB": {"p": 2}}]} + + def test_preserve_base_order_append_new(self) -> None: + """Base order preserved; new items appended at end.""" + base = {"checks": [{"A": {}}, {"B": {}}, {"C": {}}]} + override = {"checks": [{"__merge__": True}, {"D": {"new": True}}, {"B": {"updated": True}}]} + result = deep_merge(base, override) + assert result == { + "checks": [ + {"A": {}}, + {"B": {"updated": True}}, + {"C": {}}, + {"D": {"new": True}}, + ] + } + + def test_regular_lists_still_replaced(self) -> None: + """Regular lists (scalars) use replace behavior even with no marker.""" + base = {"items": [1, 2, 3]} + override = {"items": [4, 5]} + result = deep_merge(base, override) + assert result == {"items": [4, 5]} + + def test_marker_with_non_mergeable_base_strips_marker(self) -> None: + """Marker present but base not mergeable: replace with stripped list.""" + base = {"items": [{"A": {}}, "string"]} + override = {"items": [{"__merge__": True}, {"B": {}}]} + result = deep_merge(base, override) + assert result == {"items": [{"B": {}}]} + + def test_marker_with_multi_key_dicts_strips_marker(self) -> None: + """Marker present but override items have >1 key: replace with stripped list.""" + base = {"items": [{"A": 1}]} + override = {"items": [{"__merge__": True}, {"C": 3, "D": 4}]} + result = deep_merge(base, override) + assert result == {"items": [{"C": 3, "D": 4}]} + + def test_empty_base_list_with_marker(self) -> None: + """Empty base list not mergeable; marker list replaces (stripped).""" + base = {"items": []} + override = {"items": [{"__merge__": True}, {"A": {}}]} + result = deep_merge(base, override) + assert result == {"items": [{"A": {}}]} + + def test_empty_dict_values_merge(self) -> None: + """Checks with {} values merge correctly.""" + base = {"checks": [{"CheckA": {}}]} + override = {"checks": [{"__merge__": True}, {"CheckA": {"new_param": True}}]} + result = deep_merge(base, override) + assert result == {"checks": [{"CheckA": {"new_param": True}}]} + + def test_variant_names_stay_separate(self) -> None: + """Variant names like -1b and -3b are distinct keys.""" + base = { + "checks": [ + {"Workload-1b": {"gpu": 1}}, + {"Workload-3b": {"gpu": 4}}, + ] + } + override = {"checks": [{"__merge__": True}, {"Workload-1b": {"gpu": 2}}]} + result = deep_merge(base, override) + assert result == { + "checks": [ + {"Workload-1b": {"gpu": 2}}, + {"Workload-3b": {"gpu": 4}}, + ] + } + + def test_inputs_not_mutated(self) -> None: + """Original base and override are not modified.""" + base = {"checks": [{"A": {"p": 1}}, {"B": {"p": 2}}]} + override = {"checks": [{"__merge__": True}, {"A": {"p": 99}}, {"C": {"p": 3}}]} + base_copy = copy.deepcopy(base) + override_copy = copy.deepcopy(override) + deep_merge(base, override) + assert base == base_copy + assert override == override_copy + + def test_remove_nonexistent_check_is_noop(self) -> None: + """Removing a check that doesn't exist in base is a no-op.""" + base = {"checks": [{"A": {"p": 1}}]} + override = {"checks": [{"__merge__": True}, {"Z": "__remove__"}]} + result = deep_merge(base, override) + assert result == {"checks": [{"A": {"p": 1}}]} + + + def test_merge_marker_requires_true(self) -> None: + """__merge__: false should NOT trigger strategic merge.""" + base = {"checks": [{"A": {"p": 1}}, {"B": {"p": 2}}]} + override = {"checks": [{"__merge__": False}, {"A": {"p": 99}}]} + result = deep_merge(base, override) + # No strategic merge — entire list is replaced (marker stripped) + assert result == {"checks": [{"A": {"p": 99}}]} + + def test_duplicate_override_keys_deduplicated(self) -> None: + """Duplicate new keys in override list should only appear once.""" + base = {"checks": [{"A": {"p": 1}}]} + override = {"checks": [{"__merge__": True}, {"B": {"p": 2}}, {"B": {"p": 3}}]} + result = deep_merge(base, override) + # B should appear once (first occurrence wins in override_by_key) + b_items = [item for item in result["checks"] if "B" in item] + assert len(b_items) == 1 + + def test_merge_marker_stripped_when_base_key_missing(self) -> None: + """Merge marker should not leak into result when base key doesn't exist.""" + base = {"other": "value"} + override = {"checks": [{"__merge__": True}, {"A": {"p": 1}}]} + result = deep_merge(base, override) + # Marker should be stripped, only A remains + assert result == {"other": "value", "checks": [{"A": {"p": 1}}]} + + +class TestLayeredConfigs: + """Integration tests for layered template + provider configs.""" + + CONFIGS_DIR = Path(__file__).parent.parent / "configs" + + def test_template_has_no_commands(self) -> None: + """Templates should define validations only, no commands.""" + template = merge_yaml_files([self.CONFIGS_DIR / "templates" / "kaas.yaml"]) + assert "commands" not in template, "Template should not contain commands" + assert "tests" in template, "Template must contain tests" + assert "validations" in template["tests"], "Template must contain validations" + + def test_layered_merge_has_both(self) -> None: + """Template + provider merge should have both commands and tests.""" + merged = merge_yaml_files([ + self.CONFIGS_DIR / "templates" / "kaas.yaml", + self.CONFIGS_DIR / "aws" / "eks-layered.yaml", + ]) + assert "commands" in merged, "Merged config must have commands from provider" + assert "tests" in merged, "Merged config must have tests from template" + assert "kubernetes" in merged["commands"], "Commands must have kubernetes key" + steps = merged["commands"]["kubernetes"]["steps"] + assert any(s["name"] == "provision_cluster" for s in steps) + + def test_context_overrides_flow_through(self) -> None: + """Provider context values should appear in the merged config.""" + merged = merge_yaml_files([ + self.CONFIGS_DIR / "templates" / "kaas.yaml", + self.CONFIGS_DIR / "aws" / "eks-layered.yaml", + ]) + assert merged.get("context", {}).get("total_gpus") == "1" + + def test_standalone_eks_still_works(self) -> None: + """Self-contained eks.yaml should parse with both commands and tests.""" + standalone = merge_yaml_files([self.CONFIGS_DIR / "aws" / "eks.yaml"]) + assert "commands" in standalone + assert "tests" in standalone + assert "validations" in standalone["tests"] + + def test_layered_checks_match_standalone_structure(self) -> None: + """Layered and standalone should have the same validation check names.""" + standalone = merge_yaml_files([self.CONFIGS_DIR / "aws" / "eks.yaml"]) + layered = merge_yaml_files([ + self.CONFIGS_DIR / "templates" / "kaas.yaml", + self.CONFIGS_DIR / "aws" / "eks-layered.yaml", + ]) + + def get_check_names(config: dict[str, Any]) -> set[str]: + names: set[str] = set() + for group in config.get("tests", {}).get("validations", {}).values(): + for check in group: + if isinstance(check, dict): + names.update(check.keys()) + return names + + standalone_checks = get_check_names(standalone) + layered_checks = get_check_names(layered) + assert standalone_checks == layered_checks, ( + f"Check mismatch: standalone-only={standalone_checks - layered_checks}, " + f"layered-only={layered_checks - standalone_checks}" + ) + + def test_all_templates_are_validation_only(self) -> None: + """All template YAML files should have tests but no commands.""" + template_dir = self.CONFIGS_DIR / "templates" + for yaml_file in sorted(template_dir.glob("*.yaml")): + config = merge_yaml_files([yaml_file]) + assert "commands" not in config, f"{yaml_file.name} should not have commands" + assert "tests" in config, f"{yaml_file.name} must have tests" diff --git a/isvtest/src/isvtest/catalog.py b/isvtest/src/isvtest/catalog.py index 3e9ded28..da3db643 100644 --- a/isvtest/src/isvtest/catalog.py +++ b/isvtest/src/isvtest/catalog.py @@ -30,6 +30,25 @@ "NETWORK": ["templates/network.yaml"], "VM": ["templates/vm.yaml"], "IMAGE_REGISTRY": ["templates/image-registry.yaml"], + # Carbide provider — uses templates with carbidecli stubs + "CARBIDE_CONTROL_PLANE": ["carbide/control-plane.yaml"], + "CARBIDE_NETWORK": ["carbide/network.yaml"], + "CARBIDE_IMAGE_REGISTRY": ["carbide/image-registry.yaml"], + "CARBIDE_BARE_METAL": ["carbide/bm.yaml"], + "CARBIDE_IAM": ["carbide/iam.yaml"], + # OpenShift platform — platform-specific validations + "OPENSHIFT_KAAS": ["openshift/kaas.yaml"], + "OPENSHIFT_IAM": ["openshift/iam.yaml"], + "OPENSHIFT_NETWORK": ["openshift/network.yaml"], + "OPENSHIFT_STORAGE": ["openshift/storage.yaml"], + "OPENSHIFT_VM": ["openshift/vm.yaml"], + "OPENSHIFT_MACHINESET": ["openshift/machineset.yaml"], + "OPENSHIFT_SECURITY": ["openshift/security.yaml"], + "OPENSHIFT_GPU_HEALTH": ["openshift/gpu-health.yaml"], + "OPENSHIFT_DRA": ["openshift/dra.yaml"], + "OPENSHIFT_COMPUTEDOMAIN": ["openshift/computedomain.yaml"], + "OPENSHIFT_ARM": ["openshift/arm.yaml"], + "OPENSHIFT_HOSTED": ["openshift/hosted-provision.yaml"], } diff --git a/uv.lock b/uv.lock index cfc33b62..eb565962 100644 --- a/uv.lock +++ b/uv.lock @@ -661,7 +661,7 @@ requires-dist = [ { name = "pytest", specifier = ">=8.3.4,<9.0.0" }, { name = "pyyaml", specifier = ">=6.0.2,<7.0.0" }, { name = "reframe-hpc", specifier = ">=4.8.4" }, - { name = "urllib3", specifier = ">=2.6.0,<3.0.0" }, + { name = "urllib3", specifier = ">=2.6.3,<3.0.0" }, ] [[package]]