From 6101b5767f93ccb3f559f63497c096bd0388b07b Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Wed, 25 Feb 2026 17:58:53 +0100 Subject: [PATCH 1/8] feat: add opt-in strategic merge for single-key dict lists Override configs can now merge into base check lists instead of replacing them entirely, by including a `{__merge__: true}` marker. Matching checks are deep-merged by key, new checks are appended, and `"__remove__"` drops a check. Without the marker, lists are replaced as before (backward compat). Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Fabien Dupont --- isvctl/src/isvctl/config/merger.py | 109 +++++++++++++++++++- isvctl/tests/test_merger.py | 158 ++++++++++++++++++++++++++++- 2 files changed, 265 insertions(+), 2 deletions(-) diff --git a/isvctl/src/isvctl/config/merger.py b/isvctl/src/isvctl/config/merger.py index d4a200dc..1a3338ba 100644 --- a/isvctl/src/isvctl/config/merger.py +++ b/isvctl/src/isvctl/config/merger.py @@ -17,17 +17,104 @@ """ import copy +import logging from pathlib import Path from typing import Any import yaml +logger = logging.getLogger(__name__) + + +_MERGE_MARKER = "__merge__" + + +def _is_mergeable_list(lst: list[Any]) -> bool: + """Check if a list can be strategically merged. + + Returns True if the list is non-empty and every item is a dict with exactly + one key. This pattern matches config lists like checks where each item is + ``{"CheckName": {params}}``. + """ + if not lst: + return False + return all(isinstance(item, dict) and len(item) == 1 for item in lst) + + +def _has_merge_marker(lst: list[Any]) -> bool: + """Check if a list contains the ``__merge__`` opt-in marker.""" + return any( + isinstance(item, dict) and len(item) == 1 and _MERGE_MARKER in item + for item in lst + ) + + +def _strip_merge_marker(lst: list[Any]) -> list[Any]: + """Return a copy of the list with the ``__merge__`` marker removed.""" + return [ + item for item in lst + if not (isinstance(item, dict) and len(item) == 1 and _MERGE_MARKER in item) + ] + + +def _merge_single_key_dict_lists( + base_list: list[dict[str, Any]], override_list: list[dict[str, Any]] +) -> list[dict[str, Any]]: + """Merge two lists of single-key dicts by matching on the dict key. + + - Matching key: deep-merge the params (override wins on conflicts) + - New key in override: append to end + - Key not in override: keep unchanged + - Value set to ``"__remove__"``: delete the item + + Preserves base list order; new items from override appended at end. + """ + # Build an index of override items keyed by their single key + override_by_key: dict[str, Any] = {} + override_order: list[str] = [] + for item in override_list: + key = next(iter(item)) + override_by_key[key] = item[key] + override_order.append(key) + + seen_keys: set[str] = set() + result: list[dict[str, Any]] = [] + + # Walk base list, merging or removing as needed + for item in base_list: + key = next(iter(item)) + seen_keys.add(key) + + if key in override_by_key: + override_value = override_by_key[key] + if override_value == "__remove__": + continue # Drop this item + base_value = item[key] + if isinstance(base_value, dict) and isinstance(override_value, dict): + merged = deep_merge(base_value, override_value) + else: + merged = copy.deepcopy(override_value) + result.append({key: merged}) + else: + result.append(copy.deepcopy(item)) + + # Append new items from override that weren't in base + for key in override_order: + if key not in seen_keys: + if override_by_key[key] == "__remove__": + continue # Removing nonexistent key is a no-op + result.append({key: copy.deepcopy(override_by_key[key])}) + + return result + def deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: """Deep merge two dictionaries. Values from `override` take precedence. Nested dicts are merged recursively. - Lists are replaced entirely (not concatenated). + Lists are replaced entirely by default. When the override list contains a + ``{__merge__: true}`` marker and both lists are single-key dict lists, they + are merged by matching on the dict key instead. Args: base: Base dictionary @@ -42,6 +129,26 @@ def deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any] if key in result and isinstance(result[key], dict) and isinstance(value, dict): # Recursively merge nested dicts result[key] = deep_merge(result[key], value) + elif ( + key in result + and isinstance(result[key], list) + and isinstance(value, list) + and _has_merge_marker(value) + ): + # Opt-in strategic merge for lists of single-key dicts + override_items = _strip_merge_marker(value) + if _is_mergeable_list(result[key]) and _is_mergeable_list(override_items): + override_keys = [next(iter(item)) for item in override_items] + logger.debug( + "Strategic merge on '%s': %d base items, %d override items %s", + key, len(result[key]), len(override_items), override_keys, + ) + result[key] = _merge_single_key_dict_lists(result[key], override_items) + else: + logger.debug( + "Merge marker on '%s' but lists not mergeable, replacing", key, + ) + result[key] = copy.deepcopy(override_items) else: # Override with new value (including None) result[key] = copy.deepcopy(value) diff --git a/isvctl/tests/test_merger.py b/isvctl/tests/test_merger.py index 231c87aa..a804792f 100644 --- a/isvctl/tests/test_merger.py +++ b/isvctl/tests/test_merger.py @@ -10,12 +10,22 @@ """Tests for YAML merging functionality.""" +import copy from pathlib import Path from typing import Any import pytest -from isvctl.config.merger import apply_set_value, deep_merge, merge_yaml_files, parse_set_value +from isvctl.config.merger import ( + _has_merge_marker, + _is_mergeable_list, + _merge_single_key_dict_lists, + _strip_merge_marker, + apply_set_value, + deep_merge, + merge_yaml_files, + parse_set_value, +) class TestDeepMerge: @@ -183,3 +193,149 @@ def test_empty_file_ignored(self, tmp_path: Path) -> None: result = merge_yaml_files([str(file1), str(file2)]) assert result == {"a": 1} + + +class TestStrategicMerge: + """Tests for opt-in strategic merge of single-key dict lists. + + Strategic merge is triggered by including ``{__merge__: true}`` in the + override list. Without the marker, lists are replaced as before. + """ + + # -- marker helpers ------------------------------------------------------- + + def test_has_merge_marker(self) -> None: + """_has_merge_marker detects the opt-in marker.""" + assert _has_merge_marker([{"__merge__": True}]) is True + assert _has_merge_marker([{"__merge__": True}, {"A": 1}]) is True + assert _has_merge_marker([{"A": 1}]) is False + assert _has_merge_marker([]) is False + + def test_strip_merge_marker(self) -> None: + """_strip_merge_marker removes only the marker item.""" + lst = [{"__merge__": True}, {"A": 1}, {"B": 2}] + assert _strip_merge_marker(lst) == [{"A": 1}, {"B": 2}] + + def test_is_mergeable_list_helper(self) -> None: + """Direct tests for _is_mergeable_list.""" + assert _is_mergeable_list([{"A": 1}]) is True + assert _is_mergeable_list([{"A": 1}, {"B": 2}]) is True + assert _is_mergeable_list([]) is False + assert _is_mergeable_list([1, 2]) is False + assert _is_mergeable_list(["a", "b"]) is False + assert _is_mergeable_list([{"A": 1, "B": 2}]) is False + assert _is_mergeable_list([{"A": 1}, "b"]) is False + + # -- opt-in behavior via deep_merge --------------------------------------- + + def test_no_marker_replaces_list(self) -> None: + """Without __merge__, single-key dict lists are replaced (backward compat).""" + base = {"checks": [{"A": {"p": 1}}, {"B": {"p": 2}}]} + override = {"checks": [{"A": {"p": 99}}]} + result = deep_merge(base, override) + # B is gone — full replacement + assert result == {"checks": [{"A": {"p": 99}}]} + + def test_merge_matching_check_params(self) -> None: + """Core case: merge params of a matching check.""" + base = {"checks": [{"CheckA": {"param1": 1, "param2": 2}}]} + override = {"checks": [{"__merge__": True}, {"CheckA": {"param2": 99}}]} + result = deep_merge(base, override) + assert result == {"checks": [{"CheckA": {"param1": 1, "param2": 99}}]} + + def test_append_new_check(self) -> None: + """New check in override is appended to end.""" + base = {"checks": [{"CheckA": {"p": 1}}]} + override = {"checks": [{"__merge__": True}, {"CheckB": {"p": 2}}]} + result = deep_merge(base, override) + assert result == {"checks": [{"CheckA": {"p": 1}}, {"CheckB": {"p": 2}}]} + + def test_remove_check_via_sentinel(self) -> None: + """Check removed when value is '__remove__'.""" + base = {"checks": [{"CheckA": {"p": 1}}, {"CheckB": {"p": 2}}]} + override = {"checks": [{"__merge__": True}, {"CheckA": "__remove__"}]} + result = deep_merge(base, override) + assert result == {"checks": [{"CheckB": {"p": 2}}]} + + def test_preserve_base_order_append_new(self) -> None: + """Base order preserved; new items appended at end.""" + base = {"checks": [{"A": {}}, {"B": {}}, {"C": {}}]} + override = {"checks": [{"__merge__": True}, {"D": {"new": True}}, {"B": {"updated": True}}]} + result = deep_merge(base, override) + assert result == { + "checks": [ + {"A": {}}, + {"B": {"updated": True}}, + {"C": {}}, + {"D": {"new": True}}, + ] + } + + def test_regular_lists_still_replaced(self) -> None: + """Regular lists (scalars) use replace behavior even with no marker.""" + base = {"items": [1, 2, 3]} + override = {"items": [4, 5]} + result = deep_merge(base, override) + assert result == {"items": [4, 5]} + + def test_marker_with_non_mergeable_base_strips_marker(self) -> None: + """Marker present but base not mergeable: replace with stripped list.""" + base = {"items": [{"A": {}}, "string"]} + override = {"items": [{"__merge__": True}, {"B": {}}]} + result = deep_merge(base, override) + assert result == {"items": [{"B": {}}]} + + def test_marker_with_multi_key_dicts_strips_marker(self) -> None: + """Marker present but override items have >1 key: replace with stripped list.""" + base = {"items": [{"A": 1}]} + override = {"items": [{"__merge__": True}, {"C": 3, "D": 4}]} + result = deep_merge(base, override) + assert result == {"items": [{"C": 3, "D": 4}]} + + def test_empty_base_list_with_marker(self) -> None: + """Empty base list not mergeable; marker list replaces (stripped).""" + base = {"items": []} + override = {"items": [{"__merge__": True}, {"A": {}}]} + result = deep_merge(base, override) + assert result == {"items": [{"A": {}}]} + + def test_empty_dict_values_merge(self) -> None: + """Checks with {} values merge correctly.""" + base = {"checks": [{"CheckA": {}}]} + override = {"checks": [{"__merge__": True}, {"CheckA": {"new_param": True}}]} + result = deep_merge(base, override) + assert result == {"checks": [{"CheckA": {"new_param": True}}]} + + def test_variant_names_stay_separate(self) -> None: + """Variant names like -1b and -3b are distinct keys.""" + base = { + "checks": [ + {"Workload-1b": {"gpu": 1}}, + {"Workload-3b": {"gpu": 4}}, + ] + } + override = {"checks": [{"__merge__": True}, {"Workload-1b": {"gpu": 2}}]} + result = deep_merge(base, override) + assert result == { + "checks": [ + {"Workload-1b": {"gpu": 2}}, + {"Workload-3b": {"gpu": 4}}, + ] + } + + def test_inputs_not_mutated(self) -> None: + """Original base and override are not modified.""" + base = {"checks": [{"A": {"p": 1}}, {"B": {"p": 2}}]} + override = {"checks": [{"__merge__": True}, {"A": {"p": 99}}, {"C": {"p": 3}}]} + base_copy = copy.deepcopy(base) + override_copy = copy.deepcopy(override) + deep_merge(base, override) + assert base == base_copy + assert override == override_copy + + def test_remove_nonexistent_check_is_noop(self) -> None: + """Removing a check that doesn't exist in base is a no-op.""" + base = {"checks": [{"A": {"p": 1}}]} + override = {"checks": [{"__merge__": True}, {"Z": "__remove__"}]} + result = deep_merge(base, override) + assert result == {"checks": [{"A": {"p": 1}}]} From 6d240f7bf41caaf6256ac29b7da397146e21ac75 Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Wed, 25 Feb 2026 22:03:14 +0100 Subject: [PATCH 2/8] feat: support __remove__ sentinel in dict merges Keys set to "__remove__" are deleted during deep_merge, enabling selective removal of inherited values in nested dicts (e.g., dropping a single label from expected_labels without replacing the entire dict). Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Fabien Dupont --- isvctl/src/isvctl/config/merger.py | 9 ++++++--- isvctl/tests/test_merger.py | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/isvctl/src/isvctl/config/merger.py b/isvctl/src/isvctl/config/merger.py index 1a3338ba..fadfb4cc 100644 --- a/isvctl/src/isvctl/config/merger.py +++ b/isvctl/src/isvctl/config/merger.py @@ -27,6 +27,7 @@ _MERGE_MARKER = "__merge__" +_REMOVE_SENTINEL = "__remove__" def _is_mergeable_list(lst: list[Any]) -> bool: @@ -87,7 +88,7 @@ def _merge_single_key_dict_lists( if key in override_by_key: override_value = override_by_key[key] - if override_value == "__remove__": + if override_value == _REMOVE_SENTINEL: continue # Drop this item base_value = item[key] if isinstance(base_value, dict) and isinstance(override_value, dict): @@ -101,7 +102,7 @@ def _merge_single_key_dict_lists( # Append new items from override that weren't in base for key in override_order: if key not in seen_keys: - if override_by_key[key] == "__remove__": + if override_by_key[key] == _REMOVE_SENTINEL: continue # Removing nonexistent key is a no-op result.append({key: copy.deepcopy(override_by_key[key])}) @@ -126,7 +127,9 @@ def deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any] result = copy.deepcopy(base) for key, value in override.items(): - if key in result and isinstance(result[key], dict) and isinstance(value, dict): + if value == _REMOVE_SENTINEL: + result.pop(key, None) + elif key in result and isinstance(result[key], dict) and isinstance(value, dict): # Recursively merge nested dicts result[key] = deep_merge(result[key], value) elif ( diff --git a/isvctl/tests/test_merger.py b/isvctl/tests/test_merger.py index a804792f..dd66f01f 100644 --- a/isvctl/tests/test_merger.py +++ b/isvctl/tests/test_merger.py @@ -52,6 +52,27 @@ def test_list_replacement(self) -> None: result = deep_merge(base, override) assert result == {"items": [4, 5]} + def test_remove_dict_key(self) -> None: + """Test that __remove__ deletes a key during dict merge.""" + base = {"a": 1, "b": 2, "c": 3} + override = {"b": "__remove__"} + result = deep_merge(base, override) + assert result == {"a": 1, "c": 3} + + def test_remove_nested_dict_key(self) -> None: + """Test that __remove__ works inside nested dicts.""" + base = {"outer": {"a": 1, "b": 2}} + override = {"outer": {"b": "__remove__"}} + result = deep_merge(base, override) + assert result == {"outer": {"a": 1}} + + def test_remove_nonexistent_dict_key(self) -> None: + """Removing a key that doesn't exist is a no-op.""" + base = {"a": 1} + override = {"z": "__remove__"} + result = deep_merge(base, override) + assert result == {"a": 1} + def test_original_not_modified(self) -> None: """Test that original dicts are not modified.""" base = {"a": {"b": 1}} From aef1cdea74b7a3bfe420d13d67e89fda8f5062da Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Fri, 6 Mar 2026 15:16:02 +0100 Subject: [PATCH 3/8] feat: refactor templates into layered validation-only configs Templates now define WHAT to validate (checks with context variable defaults), while provider configs define HOW to provision (commands and stubs). This enables composable validation: isvctl test run -f templates/kaas.yaml -f aws/eks.yaml Key changes: - Remove commands block from all 7 templates - Replace hardcoded values with {{ context.X | default('Y') }} - Template stubs in stubs/ remain as copy-paste starting points - Existing self-contained provider configs (aws/) keep working - Update README with layered usage documentation The merge engine combines tests from the template with commands from the provider. Context variables flow through Jinja2 rendering into validation parameters at runtime. Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Fabien Dupont --- isvctl/configs/templates/README.md | 100 ++++++-- isvctl/configs/templates/bm.yaml | 253 +++---------------- isvctl/configs/templates/control-plane.yaml | 181 +------------ isvctl/configs/templates/iam.yaml | 108 +------- isvctl/configs/templates/image-registry.yaml | 192 ++------------ isvctl/configs/templates/kaas.yaml | 103 +++----- isvctl/configs/templates/network.yaml | 185 ++------------ isvctl/configs/templates/vm.yaml | 185 ++------------ 8 files changed, 229 insertions(+), 1078 deletions(-) diff --git a/isvctl/configs/templates/README.md b/isvctl/configs/templates/README.md index a6dfc946..1cba1b6a 100644 --- a/isvctl/configs/templates/README.md +++ b/isvctl/configs/templates/README.md @@ -1,38 +1,100 @@ # Validation Templates -Provider-agnostic templates for ISV Lab validation tests. Copy a template, implement the stub scripts for your platform, and run. +Provider-agnostic validation templates for ISV Lab tests. Templates define **what to validate** (checks); provider configs define **how to provision** (stubs). ## How It Works ```text -┌──────────────────┐ ┌──────────────────────┐ ┌────────────────────┐ -│ YAML Config │─────▶│ Your Stub Scripts │─────▶│ Validations │ -│ (steps + args) │ │ (call your API) │ │ (check JSON) │ -│ │ │ │ │ │ -│ You configure │ │ YOU IMPLEMENT THESE │ │ Already provided │ -│ step names, │ │ Output JSON to │ │ StepSuccessCheck, │ -│ args, timeouts │ │ stdout │ │ FieldExistsCheck │ -└──────────────────┘ └──────────────────────┘ └────────────────────┘ +┌──────────────────┐ ┌──────────────────────┐ ┌────────────────────┐ +│ Template YAML │ │ Provider YAML │ │ Validations │ +│ (validations) │ │ (commands + stubs) │ │ (check JSON) │ +│ │ │ │ │ │ +│ WHAT to check │ + │ HOW to provision │ = │ Already provided │ +│ K8sNodeReady, │ │ YOUR stub scripts │ │ StepSuccessCheck, │ +│ GpuCapacity... │ │ call your API │ │ FieldExistsCheck │ +└──────────────────┘ └──────────────────────┘ └────────────────────┘ ``` +**Two ways to use templates:** + +1. **Layered** (recommended for new providers): pair a template with a provider config + ```bash + isvctl test run -f templates/kaas.yaml -f my-provider/kaas.yaml + ``` + +2. **Self-contained** (existing approach): copy a template and add commands inline + ```bash + isvctl test run -f my-provider/kaas.yaml # has both commands + validations + ``` + **The contract is JSON.** Your scripts can be written in any language (Python, Bash, Go, etc.). They just need to print a JSON object to stdout with the required fields. ## Available Templates -| Template | Tests | Stubs | Reference Implementation | -|----------|-------|-------|--------------------------| -| `iam.yaml` | User create → verify credentials → delete | `stubs/iam/` (3 scripts) | `../stubs/aws/iam/` | -| `network.yaml` | VPC CRUD, subnets, isolation, security, connectivity, traffic | `stubs/network/` (8 scripts) | `../stubs/aws/network/` | -| `vm.yaml` | Launch GPU VM → list → reboot → NIM deploy → teardown | `stubs/vm/` (4 scripts) + `stubs/common/` (2) | `../stubs/aws/vm/` | -| `bm.yaml` | Launch bare-metal → describe → reboot → NIM → teardown → verify | `stubs/bm/` (5 scripts) + `stubs/common/` (2) | `../stubs/aws/bm/` | -| `kaas.yaml` | Provision K8s GPU cluster → validate nodes/GPU/workloads → teardown | `stubs/kaas/` (2 scripts) | `../stubs/aws/eks/` | -| `control-plane.yaml` | API health, access key lifecycle, tenant lifecycle | `stubs/control-plane/` (10 scripts) | `../stubs/aws/control-plane/` | -| `image-registry.yaml` | Image upload → VM launch → install config CRUD → BMaaS install → teardown | `stubs/image-registry/` (6 scripts) | `../stubs/aws/image-registry/` | +| Template | Validations | Provider Stubs | Reference Implementation | +|----------|-------------|----------------|--------------------------| +| `kaas.yaml` | K8s cluster health, GPU operator, GPU scheduling, workloads | `stubs/kaas/` (2 scripts) | `../aws/eks.yaml` | +| `control-plane.yaml` | API health, access key lifecycle, tenant lifecycle | `stubs/control-plane/` (10 scripts) | `../aws/control-plane.yaml` | +| `iam.yaml` | User create, verify credentials, delete | `stubs/iam/` (3 scripts) | `../aws/iam.yaml` | +| `network.yaml` | VPC CRUD, subnets, isolation, security, connectivity | `stubs/network/` (8 scripts) | `../aws/network.yaml` | +| `vm.yaml` | GPU VM lifecycle, SSH, NIM inference | `stubs/vm/` (4 scripts) + `stubs/common/` (2) | `../aws/vm.yaml` | +| `bm.yaml` | Bare-metal lifecycle, SSH, GPU, NIM | `stubs/bm/` (5 scripts) + `stubs/common/` (2) | `../aws/bm.yaml` | +| `image-registry.yaml` | Image upload, install config CRUD, BMaaS install | `stubs/image-registry/` (6 scripts) | `../aws/image-registry.yaml` | + +## Context Variables + +Templates use `{{ context.variable | default('value') }}` for provider-specific values. +Providers set these via the `context:` block in their YAML: + +```yaml +# my-provider/kaas.yaml +context: + node_count: "5" + total_gpus: "20" + gpu_per_node: "4" + gpu_operator_ns: "nvidia-gpu-operator" +``` -> **Note on Reference Implementations:** The `../stubs/aws/` paths in the "Reference Implementation" column point to NVIDIA's AWS example scripts that live _outside_ the `templates/` folder. These are optional examples provided as implementation guides — they are **not** copied when you duplicate `templates/` and are **not** required dependencies. The relative paths will not resolve once the templates folder is relocated. Refer to them in-place for inspiration, then implement your own scripts in the `stubs/` directories listed in the "Stubs" column. +Or override at runtime: +```bash +isvctl test run -f templates/kaas.yaml -f my-provider/kaas.yaml \ + --set context.node_count=10 +``` ## Quick Start +### Layered approach (new providers) + +```bash +# 1. Create a provider config with just commands + context +cat > isvctl/configs/my-isv/kaas.yaml << 'EOF' +version: "1.0" +commands: + kubernetes: + phases: ["setup", "test", "teardown"] + steps: + - name: provision_cluster + phase: setup + command: "../stubs/my-isv/kaas/setup.sh" + timeout: 1800 + - name: teardown_cluster + phase: teardown + command: "../stubs/my-isv/kaas/teardown.sh" + timeout: 1800 +context: + node_count: "3" + total_gpus: "12" +EOF + +# 2. Implement the stub scripts +vim isvctl/configs/stubs/my-isv/kaas/setup.sh + +# 3. Run with the template +isvctl test run -f isvctl/configs/templates/kaas.yaml -f isvctl/configs/my-isv/kaas.yaml +``` + +### Copy approach (existing workflow) + ```bash # 1. Copy the template folder cp -r isvctl/configs/templates/ isvctl/configs/my-isv/ diff --git a/isvctl/configs/templates/bm.yaml b/isvctl/configs/templates/bm.yaml index 3d336e8b..adf5c810 100644 --- a/isvctl/configs/templates/bm.yaml +++ b/isvctl/configs/templates/bm.yaml @@ -8,226 +8,39 @@ # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. -# Bare Metal (BMaaS) Validation - Template Configuration +# Bare Metal (BMaaS) Validation - Template # -# This is a PROVIDER-AGNOSTIC template for testing bare-metal GPU instance -# lifecycle: launch -> describe -> reboot -> NIM deploy -> teardown -> verify. -# Copy this file and the accompanying stubs/ scripts, then replace the -# script implementations with your platform's API calls. +# Provider-agnostic validations for bare-metal GPU instance lifecycle: +# launch -> describe -> reboot -> NIM deploy -> teardown -> verify. +# Pair with a provider config that supplies the commands (stubs). # -# Architecture: -# - Scripts: Platform-specific (YOU implement these) - do all BM work -# - Validations: Platform-agnostic (provided) - just check JSON output +# Usage: +# # Layered: template + provider +# isvctl test run -f isvctl/configs/templates/bm.yaml -f isvctl/configs/aws/bm.yaml # -# Contract: -# Each script MUST print a JSON object to stdout. The validations -# only inspect that JSON. As long as your scripts output the right -# fields, the validations pass regardless of which cloud you use. +# # With overrides +# isvctl test run -f ... --set context.expected_gpus=4 --set context.expected_os=rhel # -# Steps: -# 1. launch_instance (setup) - Provision bare-metal GPU instance -# 2. list_instances (test) - List instances, verify target exists -# 3. describe_instance (test) - Describe instance state + SSH info -# 4. reboot_instance (test) - Reboot instance, validate recovery -# 5. reinstall_instance (test) - Reinstall OS from stock image [skip: true by default] -# 6. deploy_nim (test) - Deploy NIM container via SSH -# 7. teardown_nim (teardown) - Stop NIM container -# 8. teardown (teardown) - Terminate instance -# 9. verify_teardown (teardown) - Confirm resources deleted -# -# Dev workflow (reuse existing instance): -# # Run 1: launch + test, keep instance alive -# BM_SKIP_TEARDOWN=true uv run isvctl test run -f isvctl/configs/my-isv/bm.yaml -# -# # Run 2+: reuse instance -# BM_INSTANCE_ID= BM_KEY_FILE=/path/to/key.pem \ -# BM_SKIP_TEARDOWN=true uv run isvctl test run -f isvctl/configs/my-isv/bm.yaml -# -# Quick start: -# 1. Copy this folder: cp -r templates/ my-isv/ -# 2. Edit the stubs: my-isv/stubs/bm/*.py -# 3. Run: uv run isvctl test run -f isvctl/configs/my-isv/bm.yaml -# -# See also: -# - AWS reference implementation: ../aws/bm.yaml + ../stubs/aws/bm/ -# - JSON schemas: isvctl/src/isvctl/config/output_schemas.py +# Context variables (set by provider or --set): +# context.expected_gpus — Number of GPUs in the instance (default: 8) +# context.expected_os — Expected OS name for SSH checks (default: ubuntu) +# context.max_reboot_uptime — Max uptime in seconds after reboot (default: 600) +# context.gpu_stress_runtime — GPU stress test duration in seconds (default: 30) +# context.gpu_stress_memory_gb — GPU stress test memory in GB (default: 16) +# context.training_steps — DDP training workload steps (default: 50) +# context.ping_target — Ethernet ping target (default: 8.8.8.8) +# context.nim_prompt — Prompt for NIM inference check (default: What is CUDA?) +# context.nim_max_tokens — Max tokens for NIM inference (default: 50) version: "1.0" -commands: - bm: - phases: ["setup", "test", "teardown"] - steps: - # ─── Step 1: Launch Bare-Metal Instance ──────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "bm", - # "instance_id": "", - # "public_ip": "", - # "key_file": "", - # "vpc_id": "", - # "instance_state": "running", - # "security_group_id": "", - # "key_name": "" - # } - - name: launch_instance - phase: setup - command: "python3 ./stubs/bm/launch_instance.py" - args: - - "--name" - - "isv-bm-test-gpu" - - "--instance-type" - - "{{instance_type}}" - - "--region" - - "{{region}}" - timeout: 1200 - - # ─── Step 2: List Instances ──────────────────────────────────── - # Reuses the VM list_instances stub (generic instance listing). - - name: list_instances - phase: test - command: "python3 ./stubs/vm/list_instances.py" - args: - - "--vpc-id" - - "{{steps.launch_instance.vpc_id}}" - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--region" - - "{{region}}" - timeout: 120 - - # ─── Step 3: Describe Instance ──────────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "bm", - # "instance_id": "...", - # "instance_state": "running", - # "public_ip": "...", - # "key_file": "..." - # } - - name: describe_instance - phase: test - command: "python3 ./stubs/bm/describe_instance.py" - args: - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--region" - - "{{region}}" - - "--key-file" - - "{{steps.launch_instance.key_file}}" - timeout: 120 - - # ─── Step 4: Reboot Instance ────────────────────────────────── - # BM-specific: longer waits for hardware POST/BIOS/OS boot. - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "bm", - # "instance_id": "...", - # "instance_state": "running", - # "public_ip": "...", - # "key_file": "...", - # "uptime_seconds": , - # "ssh_connectivity": true - # } - - name: reboot_instance - phase: test - command: "python3 ./stubs/bm/reboot_instance.py" - args: - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--region" - - "{{region}}" - - "--key-file" - - "{{steps.launch_instance.key_file}}" - - "--public-ip" - - "{{steps.launch_instance.public_ip}}" - timeout: 1800 - - # ─── Step 5: Reinstall Instance ───────────────────────────────── - # Reinstall the node from its configured stock OS. - # Skipped by default: can be very slow depending on platform. - # Set skip: false and implement stubs/bm/reinstall_instance.py to enable. - # See ../aws/bm.yaml + stubs/aws/bm/reinstall_instance.py for reference. - - name: reinstall_instance - phase: test - skip: true - command: "python3 ./stubs/bm/reinstall_instance.py" - args: - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--region" - - "{{region}}" - - "--key-file" - - "{{steps.launch_instance.key_file}}" - - "--public-ip" - - "{{steps.launch_instance.public_ip}}" - timeout: 3600 - - # ─── Step 6: Deploy NIM Container ───────────────────────────── - - name: deploy_nim - phase: test - command: "python3 ./stubs/common/deploy_nim.py" - args: - - "--host" - - "{{steps.launch_instance.public_ip}}" - - "--key-file" - - "{{steps.launch_instance.key_file}}" - timeout: 1800 - - # ─── Step 7: Teardown NIM ───────────────────────────────────── - - name: teardown_nim - phase: teardown - command: "python3 ./stubs/common/teardown_nim.py" - args: - - "--host" - - "{{steps.launch_instance.public_ip}}" - - "--key-file" - - "{{steps.launch_instance.key_file}}" - timeout: 120 - - # ─── Step 8: Teardown Instance ──────────────────────────────── - - name: teardown - phase: teardown - command: "python3 ./stubs/bm/teardown.py" - args: - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--region" - - "{{region}}" - - "--delete-key-pair" - - "--delete-security-group" - - "{{teardown_flag}}" - timeout: 1800 - - # ─── Step 9: Verify Teardown ────────────────────────────────── - # Post-teardown sanitization: verify instance terminated and - # resources deleted. - - name: verify_teardown - phase: teardown - command: "python3 ./stubs/bm/verify_terminated.py" - args: - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--region" - - "{{region}}" - - "--security-group-id" - - "{{steps.launch_instance.security_group_id}}" - - "--key-name" - - "{{steps.launch_instance.key_name}}" - timeout: 120 - tests: platform: bare_metal - cluster_name: "bm-validation" + cluster_name: "{{ context.cluster_name | default('bm-validation') }}" description: "Bare metal (BMaaS) GPU validation tests (template)" settings: - region: "us-west-2" - instance_type: "{{instance_type}}" # Supply your platform-equivalent bare-metal/GPU instance type - teardown_flag: "{{(env.BM_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" + show_skipped_tests: false # ─── Validations ───────────────────────────────────────────────── # These are PROVIDER-AGNOSTIC. They only check JSON field names/values. @@ -255,13 +68,13 @@ tests: checks: - SshConnectivityCheck: {} - SshOsCheck: - expected_os: "ubuntu" + expected_os: "{{ context.expected_os | default('ubuntu') }}" gpu: step: describe_instance checks: - SshGpuCheck: - expected_gpus: 8 + expected_gpus: "{{ context.expected_gpus | default('8') }}" host_os: step: describe_instance @@ -273,8 +86,8 @@ tests: step: describe_instance checks: - SshGpuStressCheck: - runtime: 30 - memory_gb: 16 + runtime: "{{ context.gpu_stress_runtime | default('30') }}" + memory_gb: "{{ context.gpu_stress_memory_gb | default('16') }}" # NCCL AllReduce - validates GPU-to-GPU communication (NVLink/NVSwitch) nccl: @@ -287,7 +100,7 @@ tests: step: describe_instance checks: - SshTrainingCheck: - steps: 50 + steps: "{{ context.training_steps | default('50') }}" # Network / interconnect checks nvlink: @@ -304,13 +117,13 @@ tests: step: describe_instance checks: - SshEthernetCheck: - ping_target: "8.8.8.8" + ping_target: "{{ context.ping_target | default('8.8.8.8') }}" reboot_checks: step: reboot_instance checks: - InstanceRebootCheck: - max_uptime: 600 + max_uptime: "{{ context.max_reboot_uptime | default('600') }}" reboot_state: step: reboot_instance @@ -323,13 +136,13 @@ tests: checks: - SshConnectivityCheck: {} - SshOsCheck: - expected_os: "ubuntu" + expected_os: "{{ context.expected_os | default('ubuntu') }}" reboot_gpu: step: reboot_instance checks: - SshGpuCheck: - expected_gpus: 8 + expected_gpus: "{{ context.expected_gpus | default('8') }}" reboot_host_os: step: reboot_instance @@ -348,7 +161,7 @@ tests: checks: - SshConnectivityCheck: {} - SshOsCheck: - expected_os: "ubuntu" + expected_os: "{{ context.expected_os | default('ubuntu') }}" reinstall_gpu: step: reinstall_instance @@ -369,8 +182,8 @@ tests: step: deploy_nim checks: - SshNimInferenceCheck: - prompt: "What is CUDA?" - max_tokens: 50 + prompt: "{{ context.nim_prompt | default('What is CUDA?') }}" + max_tokens: "{{ context.nim_max_tokens | default('50') }}" nim_teardown: step: teardown_nim diff --git a/isvctl/configs/templates/control-plane.yaml b/isvctl/configs/templates/control-plane.yaml index 2056a4e1..50ac7167 100644 --- a/isvctl/configs/templates/control-plane.yaml +++ b/isvctl/configs/templates/control-plane.yaml @@ -8,185 +8,32 @@ # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. -# Control Plane Validation - Template Configuration +# Control Plane Validation - Template # -# This is a PROVIDER-AGNOSTIC template for testing cloud control plane -# operations: API health, access key lifecycle, and tenant management. -# Copy this file and the accompanying stubs/ scripts, then replace the -# script implementations with your platform's API calls. +# Provider-agnostic validations for cloud control plane operations: +# API health, access key lifecycle, and tenant management. +# Pair with a provider config that supplies the commands (stubs). # -# Architecture: -# - Scripts: Platform-specific (YOU implement these) - CRUD operations -# - Validations: Platform-agnostic (provided) - just check JSON output +# Usage: +# # Layered: template + provider +# isvctl test run -f isvctl/configs/templates/control-plane.yaml -f isvctl/configs/aws/control-plane.yaml # -# Contract: -# Each script MUST print a JSON object to stdout. The validations -# only inspect that JSON. As long as your scripts output the right -# fields, the validations pass regardless of which cloud you use. +# # With overrides +# isvctl test run -f ... --set context.region=eu-west-1 # -# Test groups: -# 1. API Health - Authentication and service connectivity -# 2. Access Key Lifecycle - Create, authenticate, disable, verify rejection, delete -# 3. Tenant Lifecycle - Create, list, get info, delete -# -# Quick start: -# 1. Copy this folder: cp -r templates/ my-isv/ -# 2. Edit the stubs: my-isv/stubs/control-plane/*.py -# 3. Run: uv run isvctl test run -f isvctl/configs/my-isv/control-plane.yaml -# -# See also: -# - AWS reference implementation: ../aws/control-plane.yaml + ../stubs/aws/control-plane/ -# - JSON schemas: isvctl/src/isvctl/config/output_schemas.py +# Context variables (set by provider or --set): +# context.region — Cloud region for API calls (default: us-west-2) +# context.services — Comma-separated services to check (default: compute,storage,identity) version: "1.0" -commands: - control_plane: - phases: ["setup", "test", "teardown"] - steps: - # ═══════════════════════════════════════════════════════════════ - # SETUP: API Health Check - # ═══════════════════════════════════════════════════════════════ - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "control_plane", - # "account_id": "", - # "tests": {"auth": {"passed": true}, "service_name": {"passed": true}} - # } - - name: check_api - phase: setup - command: "python3 ./stubs/control-plane/check_api.py" - args: ["--region", "{{region}}", "--services", "{{services}}"] - timeout: 120 - - # ═══════════════════════════════════════════════════════════════ - # SETUP: Create Resources - # ═══════════════════════════════════════════════════════════════ - # Access key output: - # { - # "success": true, - # "platform": "control_plane", - # "username": "...", - # "access_key_id": "...", - # "secret_access_key": "..." - # } - - name: create_access_key - phase: setup - command: "python3 ./stubs/control-plane/create_access_key.py" - args: ["--region", "{{region}}"] - timeout: 60 - - # Tenant output: - # { - # "success": true, - # "platform": "control_plane", - # "tenant_name": "...", - # "tenant_id": "..." - # } - - name: create_tenant - phase: setup - command: "python3 ./stubs/control-plane/create_tenant.py" - args: ["--region", "{{region}}"] - timeout: 60 - - # ═══════════════════════════════════════════════════════════════ - # TEST: Access Key Lifecycle - # ═══════════════════════════════════════════════════════════════ - - name: test_access_key - phase: test - command: "python3 ./stubs/control-plane/test_access_key.py" - args: - - "--access-key-id" - - "{{steps.create_access_key.access_key_id}}" - - "--secret-access-key" - - "{{steps.create_access_key.secret_access_key}}" - - "--region" - - "{{region}}" - sensitive_args: ["--secret-access-key"] - timeout: 120 - - - name: disable_access_key - phase: test - command: "python3 ./stubs/control-plane/disable_access_key.py" - args: - - "--username" - - "{{steps.create_access_key.username}}" - - "--access-key-id" - - "{{steps.create_access_key.access_key_id}}" - - "--region" - - "{{region}}" - timeout: 60 - - - name: verify_key_rejected - phase: test - command: "python3 ./stubs/control-plane/verify_key_rejected.py" - args: - - "--access-key-id" - - "{{steps.create_access_key.access_key_id}}" - - "--secret-access-key" - - "{{steps.create_access_key.secret_access_key}}" - - "--region" - - "{{region}}" - sensitive_args: ["--secret-access-key"] - timeout: 120 - - # ═══════════════════════════════════════════════════════════════ - # TEST: Tenant Lifecycle - # ═══════════════════════════════════════════════════════════════ - - name: list_tenants - phase: test - command: "python3 ./stubs/control-plane/list_tenants.py" - args: - - "--region" - - "{{region}}" - - "--target-group" - - "{{steps.create_tenant.tenant_name}}" - timeout: 60 - - - name: get_tenant - phase: test - command: "python3 ./stubs/control-plane/get_tenant.py" - args: - - "--group-name" - - "{{steps.create_tenant.tenant_name}}" - - "--region" - - "{{region}}" - timeout: 60 - - # ═══════════════════════════════════════════════════════════════ - # TEARDOWN: Cleanup Resources - # ═══════════════════════════════════════════════════════════════ - - name: delete_access_key - phase: teardown - command: "python3 ./stubs/control-plane/delete_access_key.py" - args: - - "--username" - - "{{steps.create_access_key.username}}" - - "--access-key-id" - - "{{steps.create_access_key.access_key_id}}" - timeout: 60 - output_schema: teardown - - - name: delete_tenant - phase: teardown - command: "python3 ./stubs/control-plane/delete_tenant.py" - args: - - "--group-name" - - "{{steps.create_tenant.tenant_name}}" - - "--region" - - "{{region}}" - timeout: 60 - output_schema: teardown - tests: platform: control_plane - cluster_name: "control-plane-validation" + cluster_name: "{{ context.cluster_name | default('control-plane-validation') }}" description: "Control plane: API health, access key lifecycle, tenant lifecycle (template)" settings: - region: "us-west-2" - services: "compute,storage,identity" + show_skipped_tests: false # ─── Validations ───────────────────────────────────────────────── # These are PROVIDER-AGNOSTIC. They only check JSON field names/values. diff --git a/isvctl/configs/templates/iam.yaml b/isvctl/configs/templates/iam.yaml index 29927097..fa332d2b 100644 --- a/isvctl/configs/templates/iam.yaml +++ b/isvctl/configs/templates/iam.yaml @@ -8,117 +8,33 @@ # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. -# IAM User Lifecycle Validation - Template Configuration +# IAM User Lifecycle Validation - Template # -# This is a PROVIDER-AGNOSTIC template for testing user account -# create -> verify -> delete lifecycle. Copy this file and the -# accompanying stubs/ scripts, then replace the script implementations -# with your platform's API calls. +# Provider-agnostic validations for user account create -> verify -> delete +# lifecycle. Pair with a provider config that supplies the commands (stubs). # -# Architecture: -# - Scripts: Platform-specific (YOU implement these) - do all IAM work -# - Validations: Platform-agnostic (provided) - just check JSON output +# Usage: +# # Layered: template + provider +# isvctl test run -f isvctl/configs/templates/iam.yaml -f isvctl/configs/aws/iam.yaml # -# Contract: -# Each script MUST print a JSON object to stdout. The validations -# only inspect that JSON. As long as your scripts output the right -# fields, the validations pass regardless of which IAM system you use. +# # With overrides +# isvctl test run -f ... --set context.region=eu-west-1 # -# Steps: -# 1. create_user (setup) - Create a user account, output JSON -# 2. test_credentials (test) - Verify the credentials work, output JSON -# 3. teardown (teardown) - Delete the user, output JSON -# -# Quick start: -# 1. Copy this folder: cp -r templates/ my-isv/ -# 2. Edit the stubs: my-isv/stubs/iam/create_user.py (etc.) -# 3. Run: uv run isvctl test run -f isvctl/configs/my-isv/iam.yaml -# -# See also: -# - AWS reference implementation: ../aws/iam.yaml + ../stubs/aws/iam/ -# - JSON schemas: isvctl/src/isvctl/config/output_schemas.py +# Context variables (set by provider or --set): +# context.region — Cloud region for IAM calls (default: us-west-2) version: "1.0" -commands: - iam: - phases: ["setup", "test", "teardown"] - steps: - # ─── Step 1: Create User ─────────────────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "iam", - # "username": "", - # "user_id": "", - # "access_key_id": "", # if applicable - # "secret_access_key": "" # if applicable - # } - - name: create_user - phase: setup - command: "python3 ./stubs/iam/create_user.py" - args: - - "--username" - - "isv-test-user" - timeout: 60 - - # ─── Step 2: Test Credentials ────────────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "iam", - # "account_id": "", - # "tests": { - # "identity": {"passed": true}, - # "access": {"passed": true} - # } - # } - # - # The args below use Jinja2 templating to forward values from - # the create_user step output. Adjust arg names to match your script. - - name: test_credentials - phase: test - command: "python3 ./stubs/iam/test_credentials.py" - args: - - "--username" - - "{{steps.create_user.username}}" - - "--credential-id" - - "{{steps.create_user.access_key_id}}" - - "--credential-secret" - - "{{steps.create_user.secret_access_key}}" - sensitive_args: ["--credential-secret"] - timeout: 60 - - # ─── Step 3: Teardown (Delete User) ──────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "iam", - # "resources_deleted": ["user:"], - # "message": "User deleted successfully" - # } - - name: teardown - phase: teardown - command: "python3 ./stubs/iam/delete_user.py" - args: - - "--username" - - "{{steps.create_user.username}}" - - "{{teardown_flag}}" - timeout: 120 - tests: - cluster_name: "iam-validation" + cluster_name: "{{ context.cluster_name | default('iam-validation') }}" description: "IAM user lifecycle validation (template)" platform: iam settings: - # Override these for your environment - region: "us-west-2" - teardown_flag: "{{(env.IAM_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" + show_skipped_tests: false # ─── Validations ───────────────────────────────────────────────── # These are PROVIDER-AGNOSTIC. They only check JSON field names/values. - # You should NOT need to change these unless you add extra checks. validations: setup_checks: step: create_user diff --git a/isvctl/configs/templates/image-registry.yaml b/isvctl/configs/templates/image-registry.yaml index e8d93f77..ab1375fb 100644 --- a/isvctl/configs/templates/image-registry.yaml +++ b/isvctl/configs/templates/image-registry.yaml @@ -8,194 +8,32 @@ # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. -# Image Registry Validation - Template Configuration +# Image Registry Validation - Template # -# This is a PROVIDER-AGNOSTIC template for testing image registry lifecycle: -# - OS image management: upload, import, launch VM, teardown -# - OS install configuration CRUD (create, read, update, delete) -# - BMaaS provisioning from images and install configs +# Provider-agnostic validations for image registry lifecycle: +# OS image upload, VM launch from image, install config CRUD, +# and BMaaS provisioning from images and configs. +# Pair with a provider config that supplies the commands (stubs). # -# Architecture: -# - Scripts: Platform-specific (YOU implement these) - handle cloud operations -# - Validations: Platform-agnostic (provided) - just check JSON output -# - Schema: Uses generic field names (image_id, config_id, instance_id) +# Usage: +# # Layered: template + provider +# isvctl test run -f isvctl/configs/templates/image-registry.yaml -f isvctl/configs/aws/image-registry.yaml # -# Contract: -# Each script MUST print a JSON object to stdout. The validations -# only inspect that JSON. As long as your scripts output the right -# fields, the validations pass regardless of which cloud you use. +# # With overrides +# isvctl test run -f ... --set context.expected_os=rhel # -# Steps: -# 1. upload_image (setup) - Upload OS image to registry -# 2. launch_instance (test) - Launch VM from imported image -# 3. create_install_config (test) - CRUD an OS install configuration -# 4. install_image_bm (test) - Install OS image on bare-metal -# 5. install_config_bm (test) - Install OS config on bare-metal -# 6. teardown (teardown) - Clean up all resources -# -# Note: In the AWS reference implementation, the BM install/verify steps -# (install_image_bm, install_config_bm) are in ../aws/bm.yaml instead of -# here, as verify_image and verify_config steps within the BM lifecycle. -# -# Quick start: -# 1. Copy this folder: cp -r templates/ my-isv/ -# 2. Edit the stubs: my-isv/stubs/image-registry/*.py -# 3. Run: uv run isvctl test run -f isvctl/configs/my-isv/image-registry.yaml -# -# See also: -# - AWS reference implementation: ../aws/image-registry.yaml + ../stubs/aws/image-registry/ -# - JSON schemas: isvctl/src/isvctl/config/output_schemas.py +# Context variables (set by provider or --set): +# context.expected_os — Expected OS name for SSH checks (default: ubuntu) version: "1.0" -commands: - image_registry: - phases: ["setup", "test", "teardown"] - steps: - # ─── Step 1: Upload Image ────────────────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "image_registry", - # "image_id": "", - # "storage_bucket": "", - # "disk_ids": [""] - # } - - name: upload_image - phase: setup - command: "python3 ./stubs/image-registry/upload_image.py" - args: - - "--image-url" - - "{{image_url}}" - - "--image-format" - - "{{image_format}}" - - "--region" - - "{{region}}" - timeout: 3600 - - # ─── Step 2: Launch Instance from Image ──────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "image_registry", - # "instance_id": "", - # "public_ip": "", - # "key_path": "", - # "instance_state": "running", - # "key_name": "", - # "security_group_id": "", - # "instance_profile": "" - # } - - name: launch_instance - phase: test - command: "python3 ./stubs/image-registry/launch_instance.py" - args: - - "--image-id" - - "{{steps.upload_image.image_id}}" - - "--instance-type" - - "{{instance_type}}" - - "--region" - - "{{region}}" - timeout: 600 - - # ─── Step 3: CRUD OS Install Configuration ──────────────────── - # YOUR SCRIPT must create, read, update, and delete an OS - # install configuration (e.g., iPXE config, Carbide profile). - # { - # "success": true, - # "platform": "image_registry", - # "config_id": "", - # "config_name": "", - # "operations": { - # "create": {"passed": true}, - # "read": {"passed": true}, - # "update": {"passed": true}, - # "delete": {"passed": true} - # } - # } - - name: crud_install_config - phase: test - command: "python3 ./stubs/image-registry/crud_install_config.py" - args: - - "--region" - - "{{region}}" - timeout: 300 - - # ─── Step 4: Install OS Image on Bare-Metal ────────────────── - # YOUR SCRIPT must provision a BM node from the uploaded OS image. - # { - # "success": true, - # "platform": "image_registry", - # "instance_id": "", - # "image_id": "", - # "instance_state": "running" - # } - - name: install_image_bm - phase: test - command: "python3 ./stubs/image-registry/install_image_bm.py" - args: - - "--image-id" - - "{{steps.upload_image.image_id}}" - - "--region" - - "{{region}}" - timeout: 1800 - - # ─── Step 5: Install OS Config on Bare-Metal ───────────────── - # YOUR SCRIPT must provision a BM node using an install config. - # { - # "success": true, - # "platform": "image_registry", - # "instance_id": "", - # "config_id": "", - # "instance_state": "running" - # } - - name: install_config_bm - phase: test - command: "python3 ./stubs/image-registry/install_config_bm.py" - args: - - "--config-id" - - "{{steps.crud_install_config.config_id}}" - - "--region" - - "{{region}}" - timeout: 1800 - - # ─── Step 6: Teardown ────────────────────────────────────────── - # YOUR SCRIPT must clean up all resources: instances, images, - # configs, storage, keys, security groups. - - name: teardown - phase: teardown - command: "python3 ./stubs/image-registry/teardown.py" - args: - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--image-id" - - "{{steps.upload_image.image_id}}" - - "--disk-ids" - - "{{steps.upload_image.disk_ids | join(',')}}" - - "--bucket-name" - - "{{steps.upload_image.storage_bucket}}" - - "--key-name" - - "{{steps.launch_instance.key_name}}" - - "--security-group-id" - - "{{steps.launch_instance.security_group_id}}" - - "--instance-profile" - - "{{steps.launch_instance.instance_profile}}" - - "--region" - - "{{region}}" - - "{{teardown_flag}}" - timeout: 1800 - tests: - cluster_name: "image-registry-validation" + cluster_name: "{{ context.cluster_name | default('image-registry-validation') }}" description: "Image registry validation tests (template)" platform: image_registry settings: - region: "us-west-2" - image_url: "https://cloud-images.ubuntu.com/releases/noble/release/ubuntu-24.04-server-cloudimg-amd64.vmdk" - image_format: "vmdk" - instance_type: "g4dn.xlarge" - teardown_flag: "{{(env.IR_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" + show_skipped_tests: false # ─── Validations ───────────────────────────────────────────────── # These are PROVIDER-AGNOSTIC. They only check JSON field names/values. @@ -223,7 +61,7 @@ tests: checks: - SshConnectivityCheck: {} - SshOsCheck: - expected_os: "ubuntu" + expected_os: "{{ context.expected_os | default('ubuntu') }}" # --- OS Install Config CRUD --- install_config_crud: diff --git a/isvctl/configs/templates/kaas.yaml b/isvctl/configs/templates/kaas.yaml index a0d89caf..5c668f57 100644 --- a/isvctl/configs/templates/kaas.yaml +++ b/isvctl/configs/templates/kaas.yaml @@ -8,104 +8,61 @@ # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. -# Kubernetes-as-a-Service (KaaS) Cluster Validation - Template Configuration +# Kubernetes-as-a-Service (KaaS) Cluster Validation - Template # -# This is a PROVIDER-AGNOSTIC template for testing GPU-enabled Kubernetes -# cluster lifecycle: provision -> validate -> workloads -> teardown. -# Copy this file and the accompanying stubs/ scripts, then replace the -# script implementations with your platform's provisioning logic. +# Provider-agnostic validations for GPU-enabled Kubernetes clusters. +# Pair with a provider config that supplies the commands (stubs). # -# Architecture: -# - Scripts: Platform-specific (YOU implement these) - provision/teardown cluster -# - Validations: Platform-agnostic (provided) - Kubernetes API checks +# Usage: +# # Layered: template + provider +# isvctl test run -f isvctl/configs/templates/kaas.yaml -f isvctl/configs/aws/eks.yaml # -# Contract: -# Setup script must configure kubectl access (e.g., write kubeconfig) -# and output a JSON object to stdout. Validations use kubectl directly. +# # With overrides +# isvctl test run -f ... --set context.node_count=5 --set context.total_gpus=4 # -# Steps: -# 1. provision_cluster (setup) - Provision K8s cluster, configure kubectl -# 2. teardown_cluster (teardown) - Destroy cluster and resources -# -# Quick start: -# 1. Copy this folder: cp -r templates/ my-isv/ -# 2. Edit the stubs: my-isv/stubs/kaas/setup.sh, teardown.sh -# 3. Run: uv run isvctl test run -f isvctl/configs/my-isv/kaas.yaml -# -# See also: -# - AWS reference implementation: ../aws/eks.yaml + ../stubs/aws/eks/ -# - Terraform example: ../stubs/aws/eks/terraform/ +# Context variables (set by provider or --set): +# context.node_count — Expected node count (default: 3) +# context.gpu_per_node — GPUs per node (default: 1) +# context.total_gpus — Total GPUs in cluster (default: 3) +# context.gpu_node_count — Number of GPU nodes (default: 1) +# context.runtime_class — RuntimeClass for GPU pods (default: nvidia) +# context.driver_version — Expected NVIDIA driver version (default: 570.195.03) +# context.gpu_operator_ns — GPU Operator namespace (default: gpu-operator) +# context.min_nccl_bw — Min NCCL bus bandwidth in Gbps (default: 100) version: "1.0" -commands: - kubernetes: - phases: ["setup", "test", "teardown"] - steps: - # ─── Setup: Provision Cluster ────────────────────────────────── - # YOUR SCRIPT must: - # 1. Provision a Kubernetes cluster with GPU nodes - # 2. Configure kubectl (write kubeconfig) - # 3. Print JSON to stdout: - # { - # "success": true, - # "platform": "kubernetes", - # "cluster_name": "", - # "cluster_endpoint": "", - # "node_count": , - # "gpu_node_count": - # } - - name: provision_cluster - phase: setup - command: "./stubs/kaas/setup.sh" - timeout: 1800 - env: - CLUSTER_REGION: "us-west-2" - - # ─── Teardown: Destroy Cluster ──────────────────────────────── - # YOUR SCRIPT must destroy all cluster resources. - - name: teardown_cluster - phase: teardown - command: "./stubs/kaas/teardown.sh" - timeout: 1800 - env: - # Override with TEARDOWN_ENABLED=false to preserve resources for debugging - TEARDOWN_ENABLED: "true" - tests: - cluster_name: "{{inventory.cluster_name}}" - description: "Kubernetes GPU cluster validation (template)" + cluster_name: "{{ context.cluster_name | default('') }}" + description: "Kubernetes GPU cluster validation" platform: kubernetes settings: show_skipped_tests: false - # ─── Validations ───────────────────────────────────────────────── - # These use kubectl (configured by setup script) to check cluster state. - # Adjust counts and settings to match your cluster configuration. validations: kubernetes: - K8sNodeCountCheck: - count: "3" + count: "{{ context.node_count | default('3') }}" - K8sNodeReadyCheck: require_all_ready: true - K8sNvidiaSmiCheck: - runtime_class: "nvidia" + runtime_class: "{{ context.runtime_class | default('nvidia') }}" - K8sDriverVersionCheck: - driver_version: "570.195.03" - runtime_class: "nvidia" + driver_version: "{{ context.driver_version | default('570.195.03') }}" + runtime_class: "{{ context.runtime_class | default('nvidia') }}" - K8sGpuPodAccessCheck: gpu_count: 1 - total_gpu_count: 1 - runtime_class: "nvidia" + total_gpu_count: "{{ context.gpu_node_count | default('1') }}" + runtime_class: "{{ context.runtime_class | default('nvidia') }}" - K8sGpuCapacityCheck: resource_name: "nvidia.com/gpu" - expected_per_node: 1 - expected_total: 3 + expected_per_node: "{{ context.gpu_per_node | default('1') }}" + expected_total: "{{ context.total_gpus | default('3') }}" - K8sGpuOperatorNamespaceCheck: - namespace: "gpu-operator" + namespace: "{{ context.gpu_operator_ns | default('gpu-operator') }}" - K8sGpuOperatorPodsCheck: - namespace: "gpu-operator" + namespace: "{{ context.gpu_operator_ns | default('gpu-operator') }}" - K8sGpuLabelsCheck: label_selector: "nvidia.com/gpu.present=true" - K8sPodHealthCheck: @@ -116,7 +73,7 @@ tests: k8s_workloads: - K8sNcclWorkload: - min_bus_bw_gbps: 100 + min_bus_bw_gbps: "{{ context.min_nccl_bw | default('100') }}" - K8sGpuStressWorkload: memory_gb: 1 runtime: 30 diff --git a/isvctl/configs/templates/network.yaml b/isvctl/configs/templates/network.yaml index 4fdabc83..6e3a7f2b 100644 --- a/isvctl/configs/templates/network.yaml +++ b/isvctl/configs/templates/network.yaml @@ -8,183 +8,34 @@ # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. -# Network Validation - Template Configuration +# Network Validation - Template # -# This is a PROVIDER-AGNOSTIC template for comprehensive network testing. -# Copy this file and the accompanying stubs/ scripts, then replace the -# script implementations with your platform's API calls. +# Provider-agnostic validations for comprehensive network testing: +# VPC CRUD, subnet configuration, VPC isolation, security blocking, +# connectivity, and traffic validation. +# Pair with a provider config that supplies the commands (stubs). # -# Architecture: -# - Scripts: Platform-specific (YOU implement these) - do all network work -# - Validations: Platform-agnostic (provided) - just check JSON output +# Usage: +# # Layered: template + provider +# isvctl test run -f isvctl/configs/templates/network.yaml -f isvctl/configs/aws/network.yaml # -# Contract: -# Each script MUST print a JSON object to stdout. The validations -# only inspect that JSON. As long as your scripts output the right -# fields, the validations pass regardless of which cloud you use. +# # With overrides +# isvctl test run -f ... --set context.region=eu-west-1 --set context.min_subnets=3 # -# Test suites: -# 1. VPC CRUD - Create, Read, Update, Delete lifecycle -# 2. Subnet Configuration - Multi-AZ subnet distribution -# 3. VPC Isolation - Security boundaries between VPCs -# 4. Security Blocking - Firewall / ACL blocking rules (negative tests) -# 5. Connectivity - Instance network assignment -# 6. Traffic Validation - Real ping tests between instances -# -# Quick start: -# 1. Copy this folder: cp -r templates/ my-isv/ -# 2. Edit the stubs: my-isv/stubs/network/*.py -# 3. Run: uv run isvctl test run -f isvctl/configs/my-isv/network.yaml -# -# See also: -# - AWS reference implementation: ../aws/network.yaml + ../stubs/aws/network/ -# - JSON schemas: isvctl/src/isvctl/config/output_schemas.py +# Context variables (set by provider or --set): +# context.region — Cloud region for network calls (default: us-west-2) +# context.min_subnets — Minimum subnets for setup check (default: 2) +# context.subnet_count — Expected subnet count for subnet config test (default: 4) version: "1.0" -commands: - network: - phases: ["setup", "test", "teardown"] - steps: - # ─── Setup: Create Shared VPC ────────────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "network", - # "network_id": "", - # "subnets": [{"subnet_id": "..."}], - # "security_group_id": "" - # } - - name: create_network - phase: setup - command: "python3 ./stubs/network/create_vpc.py" - args: - - "--name" - - "isv-shared-vpc" - - "--region" - - "{{region}}" - - "--cidr" - - "10.0.0.0/16" - timeout: 300 - - # ─── Test 1: VPC CRUD Operations ─────────────────────────────── - # Self-contained test. Creates its own VPC, tests CRUD, cleans up. - # YOUR SCRIPT must output JSON matching the vpc_crud schema. - - name: vpc_crud - phase: test - command: "python3 ./stubs/network/vpc_crud_test.py" - args: - - "--region" - - "{{region}}" - - "--cidr" - - "10.99.0.0/16" - timeout: 120 - output_schema: vpc_crud - - # ─── Test 2: Subnet Configuration ────────────────────────────── - # Self-contained test. Creates VPC with multiple subnets, verifies. - # YOUR SCRIPT must output JSON matching the subnet_config schema. - - name: subnet_config - phase: test - command: "python3 ./stubs/network/subnet_test.py" - args: - - "--region" - - "{{region}}" - - "--cidr" - - "10.98.0.0/16" - - "--subnet-count" - - "4" - timeout: 120 - output_schema: subnet_config - - # ─── Test 3: VPC Isolation ───────────────────────────────────── - # Self-contained test. Creates two VPCs, verifies no cross-access. - # YOUR SCRIPT must output JSON matching the vpc_isolation schema. - - name: vpc_isolation - phase: test - command: "python3 ./stubs/network/isolation_test.py" - args: - - "--region" - - "{{region}}" - - "--cidr-a" - - "10.97.0.0/16" - - "--cidr-b" - - "10.96.0.0/16" - timeout: 120 - output_schema: vpc_isolation - - # ─── Test 4: Security Blocking ───────────────────────────────── - # Self-contained test. Tests firewall/ACL blocking rules. - # YOUR SCRIPT must output JSON matching the security_blocking schema. - - name: security_blocking - phase: test - command: "python3 ./stubs/network/security_test.py" - args: - - "--region" - - "{{region}}" - - "--cidr" - - "10.94.0.0/16" - timeout: 120 - output_schema: security_blocking - - # ─── Test 5: Connectivity ────────────────────────────────────── - # Uses the shared VPC from create_network step. - # YOUR SCRIPT must output JSON matching the connectivity_result schema. - - name: connectivity_test - phase: test - command: "python3 ./stubs/network/test_connectivity.py" - args: - - "--vpc-id" - - "{{steps.create_network.network_id}}" - - "--subnet-ids" - - "{{steps.create_network.subnets | map(attribute='subnet_id') | join(',')}}" - - "--sg-id" - - "{{steps.create_network.security_group_id}}" - - "--region" - - "{{region}}" - timeout: 600 - output_schema: connectivity_result - - # ─── Test 6: Traffic Validation ──────────────────────────────── - # Self-contained test. Real traffic tests between instances. - # YOUR SCRIPT must output JSON matching the traffic_flow schema. - - name: traffic_validation - phase: test - command: "python3 ./stubs/network/traffic_test.py" - args: - - "--region" - - "{{region}}" - - "--cidr" - - "10.93.0.0/16" - timeout: 900 - output_schema: traffic_flow - - # ─── Teardown: Clean up shared VPC ───────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "network", - # "resources_deleted": [...], - # "message": "..." - # } - - name: teardown - phase: teardown - command: "python3 ./stubs/network/teardown.py" - args: - - "--vpc-id" - - "{{steps.create_network.network_id}}" - - "--region" - - "{{region}}" - timeout: 300 - output_schema: teardown - tests: - cluster_name: "network-validation" + cluster_name: "{{ context.cluster_name | default('network-validation') }}" description: "Network validation tests (template)" platform: network settings: - region: "us-west-2" + show_skipped_tests: false # ─── Validations ───────────────────────────────────────────────── # These are PROVIDER-AGNOSTIC. They only check JSON field names/values. @@ -194,7 +45,7 @@ tests: checks: - NetworkProvisionedCheck: require_subnets: true - min_subnets: 2 + min_subnets: "{{ context.min_subnets | default('2') }}" network: checks: @@ -202,7 +53,7 @@ tests: step: vpc_crud - SubnetConfigCheck: step: subnet_config - min_subnets: 4 + min_subnets: "{{ context.subnet_count | default('4') }}" require_multi_az: true - VpcIsolationCheck: step: vpc_isolation diff --git a/isvctl/configs/templates/vm.yaml b/isvctl/configs/templates/vm.yaml index 31fff2c7..3ef3ff0b 100644 --- a/isvctl/configs/templates/vm.yaml +++ b/isvctl/configs/templates/vm.yaml @@ -8,168 +8,35 @@ # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. -# VM (Virtual Machine) Validation - Template Configuration +# VM (Virtual Machine) Validation - Template # -# This is a PROVIDER-AGNOSTIC template for testing GPU virtual machine -# lifecycle: launch -> verify -> reboot -> NIM deploy -> teardown. -# Copy this file and the accompanying stubs/ scripts, then replace the -# script implementations with your platform's API calls. +# Provider-agnostic validations for GPU virtual machine lifecycle: +# launch -> verify -> reboot -> NIM deploy -> teardown. +# Pair with a provider config that supplies the commands (stubs). # -# Architecture: -# - Scripts: Platform-specific (YOU implement these) - do all VM work -# - Validations: Platform-agnostic (provided) - just check JSON output +# Usage: +# # Layered: template + provider +# isvctl test run -f isvctl/configs/templates/vm.yaml -f isvctl/configs/aws/vm.yaml # -# Contract: -# Each script MUST print a JSON object to stdout. The validations -# only inspect that JSON. As long as your scripts output the right -# fields, the validations pass regardless of which cloud you use. +# # With overrides +# isvctl test run -f ... --set context.expected_gpus=4 --set context.expected_os=rhel # -# Steps: -# 1. launch_instance (setup) - Provision GPU VM, output IP/key -# 2. list_instances (test) - List instances, verify target exists -# 3. reboot_instance (test) - Reboot VM, validate recovery -# 4. deploy_nim (test) - Deploy NIM container via SSH -# 5. teardown_nim (teardown) - Stop NIM container via SSH -# 6. teardown (teardown) - Terminate VM and cleanup -# -# Quick start: -# 1. Copy this folder: cp -r templates/ my-isv/ -# 2. Edit the stubs: my-isv/stubs/vm/*.py -# 3. Run: uv run isvctl test run -f isvctl/configs/my-isv/vm.yaml -# -# See also: -# - AWS reference implementation: ../aws/vm.yaml + ../stubs/aws/vm/ -# - JSON schemas: isvctl/src/isvctl/config/output_schemas.py +# Context variables (set by provider or --set): +# context.expected_gpus — Number of GPUs in the VM (default: 1) +# context.expected_os — Expected OS name for SSH checks (default: ubuntu) +# context.max_reboot_uptime — Max uptime in seconds after reboot (default: 600) +# context.nim_prompt — Prompt for NIM inference check (default: What is CUDA?) +# context.nim_max_tokens — Max tokens for NIM inference (default: 50) version: "1.0" -commands: - vm: - phases: ["setup", "test", "teardown"] - steps: - # ─── Step 1: Launch Instance ─────────────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "vm", - # "instance_id": "", - # "public_ip": "", - # "key_file": "", - # "vpc_id": "", - # "instance_state": "running", - # "security_group_id": "", - # "key_name": "" - # } - - name: launch_instance - phase: setup - command: "python3 ./stubs/vm/launch_instance.py" - args: - - "--name" - - "isv-test-gpu" - - "--instance-type" - - "{{instance_type}}" - - "--region" - - "{{region}}" - timeout: 600 - - # ─── Step 2: List Instances ──────────────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "vm", - # "instances": [{"instance_id": "...", "state": "running"}], - # "total_count": 1 - # } - - name: list_instances - phase: test - command: "python3 ./stubs/vm/list_instances.py" - args: - - "--vpc-id" - - "{{steps.launch_instance.vpc_id}}" - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--region" - - "{{region}}" - timeout: 120 - - # ─── Step 3: Reboot Instance ────────────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "vm", - # "instance_id": "...", - # "instance_state": "running", - # "public_ip": "...", - # "key_file": "...", - # "uptime_seconds": , - # "ssh_connectivity": true - # } - - name: reboot_instance - phase: test - command: "python3 ./stubs/vm/reboot_instance.py" - args: - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--region" - - "{{region}}" - - "--key-file" - - "{{steps.launch_instance.key_file}}" - - "--public-ip" - - "{{steps.launch_instance.public_ip}}" - timeout: 600 - - # ─── Step 4: Deploy NIM Container ───────────────────────────── - # Shared script - deploys NIM inference container via SSH. - # See stubs/common/deploy_nim.py for the JSON contract. - - name: deploy_nim - phase: test - command: "python3 ./stubs/common/deploy_nim.py" - args: - - "--host" - - "{{steps.launch_instance.public_ip}}" - - "--key-file" - - "{{steps.launch_instance.key_file}}" - timeout: 1800 - - # ─── Step 5: Teardown NIM ───────────────────────────────────── - - name: teardown_nim - phase: teardown - command: "python3 ./stubs/common/teardown_nim.py" - args: - - "--host" - - "{{steps.launch_instance.public_ip}}" - - "--key-file" - - "{{steps.launch_instance.key_file}}" - timeout: 120 - - # ─── Step 6: Teardown Instance ──────────────────────────────── - # YOUR SCRIPT must output JSON with at minimum: - # { - # "success": true, - # "platform": "vm", - # "resources_deleted": [...], - # "message": "..." - # } - - name: teardown - phase: teardown - command: "python3 ./stubs/vm/teardown.py" - args: - - "--instance-id" - - "{{steps.launch_instance.instance_id}}" - - "--region" - - "{{region}}" - - "--delete-key-pair" - - "--delete-security-group" - timeout: 600 - tests: platform: vm - cluster_name: "vm-validation" + cluster_name: "{{ context.cluster_name | default('vm-validation') }}" description: "VM (virtual machine) GPU validation tests (template)" settings: - region: "us-west-2" - instance_type: "g5.xlarge" + show_skipped_tests: false # ─── Validations ───────────────────────────────────────────────── # These are PROVIDER-AGNOSTIC. They only check JSON field names/values. @@ -191,27 +58,27 @@ tests: checks: - SshConnectivityCheck: {} - SshOsCheck: - expected_os: "ubuntu" + expected_os: "{{ context.expected_os | default('ubuntu') }}" gpu: step: launch_instance checks: - SshGpuCheck: - expected_gpus: 1 + expected_gpus: "{{ context.expected_gpus | default('1') }}" host_os: step: launch_instance checks: - SshVcpuPinningCheck: {} - SshPciBusCheck: - expected_gpus: 1 + expected_gpus: "{{ context.expected_gpus | default('1') }}" - SshHostSoftwareCheck: {} reboot_checks: step: reboot_instance checks: - InstanceRebootCheck: - max_uptime: 600 + max_uptime: "{{ context.max_reboot_uptime | default('600') }}" reboot_state: step: reboot_instance @@ -224,20 +91,20 @@ tests: checks: - SshConnectivityCheck: {} - SshOsCheck: - expected_os: "ubuntu" + expected_os: "{{ context.expected_os | default('ubuntu') }}" reboot_gpu: step: reboot_instance checks: - SshGpuCheck: - expected_gpus: 1 + expected_gpus: "{{ context.expected_gpus | default('1') }}" reboot_host_os: step: reboot_instance checks: - SshVcpuPinningCheck: {} - SshPciBusCheck: - expected_gpus: 1 + expected_gpus: "{{ context.expected_gpus | default('1') }}" - SshHostSoftwareCheck: {} nim_health: @@ -254,8 +121,8 @@ tests: step: deploy_nim checks: - SshNimInferenceCheck: - prompt: "What is CUDA?" - max_tokens: 50 + prompt: "{{ context.nim_prompt | default('What is CUDA?') }}" + max_tokens: "{{ context.nim_max_tokens | default('50') }}" nim_teardown: step: teardown_nim From 44e861963beaa2dfb0039ff3d145f73c1256bc0c Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Fri, 6 Mar 2026 15:39:50 +0100 Subject: [PATCH 4/8] feat: add AWS EKS layered config as reference implementation Adds eks-layered.yaml that supplies only commands (Terraform stubs) and context overrides, designed to pair with templates/kaas.yaml: isvctl test run \ -f isvctl/configs/templates/kaas.yaml \ -f isvctl/configs/aws/eks-layered.yaml The existing self-contained eks.yaml is unchanged (backward compat). Adds 6 integration tests verifying: - Templates have no commands block - Layered merge produces both commands and tests - Context overrides flow through - Standalone eks.yaml still works - Layered and standalone have the same validation check names - All 7 templates are validation-only Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Fabien Dupont --- isvctl/configs/aws/eks-layered.yaml | 71 ++++++++++++++++ isvctl/configs/templates/kaas.yaml | 2 +- isvctl/src/isvctl/config/merger.py | 22 +++-- isvctl/tests/test_merger.py | 122 ++++++++++++++++++++++++---- uv.lock | 2 +- 5 files changed, 195 insertions(+), 24 deletions(-) create mode 100644 isvctl/configs/aws/eks-layered.yaml diff --git a/isvctl/configs/aws/eks-layered.yaml b/isvctl/configs/aws/eks-layered.yaml new file mode 100644 index 00000000..acf6b981 --- /dev/null +++ b/isvctl/configs/aws/eks-layered.yaml @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# AWS EKS - Layered Provider Configuration +# +# Supplies commands (Terraform stubs) and context overrides for the +# kaas template. Use with the template for composable validation: +# +# isvctl test run \ +# -f isvctl/configs/templates/kaas.yaml \ +# -f isvctl/configs/aws/eks-layered.yaml +# +# For a self-contained config (no template needed), use eks.yaml instead. +# +# Environment Variables: +# TF_AUTO_APPROVE — Skip Terraform approval prompts (default: false) +# TF_VAR_* — Terraform variables (region, instance types, etc.) +# AWS_SKIP_TEARDOWN — Preserve resources after test (default: false) +# NGC_API_KEY — NGC API key for NIM workloads +# AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY — AWS credentials + +version: "1.0" + +# ============================================================================= +# Provider-specific commands (Terraform stubs) +# ============================================================================= +commands: + kubernetes: + phases: ["setup", "test", "teardown"] + steps: + - name: provision_cluster + phase: setup + command: "../stubs/aws/eks/setup.sh" + timeout: 1800 + env: + TF_AUTO_APPROVE: "true" + TF_VAR_region: "us-west-2" + TF_VAR_gpu_node_instance_types: '["g5.xlarge"]' + TF_VAR_gpu_node_desired_size: "1" + # Override with your IP CIDR for security (e.g., '["203.0.113.0/24"]') + # TF_VAR_cluster_endpoint_public_access_cidrs: '["0.0.0.0/0"]' + + - name: teardown_cluster + phase: teardown + command: "../stubs/aws/eks/teardown.sh" + timeout: 1800 + env: + TF_AUTO_APPROVE: "true" + +# ============================================================================= +# Context overrides for template defaults +# ============================================================================= +# Single g5.xlarge GPU node = 1 GPU total (template defaults to 3) +context: + total_gpus: "1" + +# ============================================================================= +# Validation overrides (optional) +# ============================================================================= +# The template provides all checks with sensible defaults. Override only +# what differs for this provider. Use {__merge__: true} to merge into +# the template's check list instead of replacing it. +tests: + description: "AWS EKS GPU cluster validation" diff --git a/isvctl/configs/templates/kaas.yaml b/isvctl/configs/templates/kaas.yaml index 5c668f57..7cd5b0ae 100644 --- a/isvctl/configs/templates/kaas.yaml +++ b/isvctl/configs/templates/kaas.yaml @@ -33,7 +33,7 @@ version: "1.0" tests: - cluster_name: "{{ context.cluster_name | default('') }}" + cluster_name: "{{ context.cluster_name | default(inventory.cluster_name | default('')) }}" description: "Kubernetes GPU cluster validation" platform: kubernetes diff --git a/isvctl/src/isvctl/config/merger.py b/isvctl/src/isvctl/config/merger.py index fadfb4cc..016f1479 100644 --- a/isvctl/src/isvctl/config/merger.py +++ b/isvctl/src/isvctl/config/merger.py @@ -43,9 +43,13 @@ def _is_mergeable_list(lst: list[Any]) -> bool: def _has_merge_marker(lst: list[Any]) -> bool: - """Check if a list contains the ``__merge__`` opt-in marker.""" + """Check if a list contains the ``{__merge__: true}`` opt-in marker. + + Only the exact shape ``{"__merge__": True}`` triggers strategic merge. + """ return any( - isinstance(item, dict) and len(item) == 1 and _MERGE_MARKER in item + isinstance(item, dict) and len(item) == 1 + and item.get(_MERGE_MARKER) is True for item in lst ) @@ -99,9 +103,10 @@ def _merge_single_key_dict_lists( else: result.append(copy.deepcopy(item)) - # Append new items from override that weren't in base + # Append new items from override that weren't in base (deduplicated) for key in override_order: if key not in seen_keys: + seen_keys.add(key) if override_by_key[key] == _REMOVE_SENTINEL: continue # Removing nonexistent key is a no-op result.append({key: copy.deepcopy(override_by_key[key])}) @@ -127,9 +132,7 @@ def deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any] result = copy.deepcopy(base) for key, value in override.items(): - if value == _REMOVE_SENTINEL: - result.pop(key, None) - elif key in result and isinstance(result[key], dict) and isinstance(value, dict): + if key in result and isinstance(result[key], dict) and isinstance(value, dict): # Recursively merge nested dicts result[key] = deep_merge(result[key], value) elif ( @@ -154,7 +157,12 @@ def deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any] result[key] = copy.deepcopy(override_items) else: # Override with new value (including None) - result[key] = copy.deepcopy(value) + # Strip any __merge__ markers from lists to avoid leaking into runtime config + if isinstance(value, list): + stripped = _strip_merge_marker(value) + result[key] = copy.deepcopy(stripped) + else: + result[key] = copy.deepcopy(value) return result diff --git a/isvctl/tests/test_merger.py b/isvctl/tests/test_merger.py index dd66f01f..c0bddd58 100644 --- a/isvctl/tests/test_merger.py +++ b/isvctl/tests/test_merger.py @@ -52,26 +52,21 @@ def test_list_replacement(self) -> None: result = deep_merge(base, override) assert result == {"items": [4, 5]} - def test_remove_dict_key(self) -> None: - """Test that __remove__ deletes a key during dict merge.""" + def test_remove_string_is_literal_at_dict_level(self) -> None: + """__remove__ at dict level is treated as a literal string value.""" base = {"a": 1, "b": 2, "c": 3} override = {"b": "__remove__"} result = deep_merge(base, override) - assert result == {"a": 1, "c": 3} + # __remove__ only works inside strategic-merge lists, not at dict level + assert result == {"a": 1, "b": "__remove__", "c": 3} - def test_remove_nested_dict_key(self) -> None: - """Test that __remove__ works inside nested dicts.""" - base = {"outer": {"a": 1, "b": 2}} - override = {"outer": {"b": "__remove__"}} - result = deep_merge(base, override) - assert result == {"outer": {"a": 1}} - - def test_remove_nonexistent_dict_key(self) -> None: - """Removing a key that doesn't exist is a no-op.""" - base = {"a": 1} - override = {"z": "__remove__"} + def test_remove_only_in_strategic_merge_lists(self) -> None: + """__remove__ deletes items only within strategic-merge lists.""" + base = {"checks": [{"A": {"p": 1}}, {"B": {"p": 2}}]} + override = {"checks": [{"__merge__": True}, {"B": "__remove__"}]} result = deep_merge(base, override) - assert result == {"a": 1} + # B is removed from the list, A is kept + assert result == {"checks": [{"A": {"p": 1}}]} def test_original_not_modified(self) -> None: """Test that original dicts are not modified.""" @@ -360,3 +355,100 @@ def test_remove_nonexistent_check_is_noop(self) -> None: override = {"checks": [{"__merge__": True}, {"Z": "__remove__"}]} result = deep_merge(base, override) assert result == {"checks": [{"A": {"p": 1}}]} + + + def test_merge_marker_requires_true(self) -> None: + """__merge__: false should NOT trigger strategic merge.""" + base = {"checks": [{"A": {"p": 1}}, {"B": {"p": 2}}]} + override = {"checks": [{"__merge__": False}, {"A": {"p": 99}}]} + result = deep_merge(base, override) + # No strategic merge — entire list is replaced (marker stripped) + assert result == {"checks": [{"A": {"p": 99}}]} + + def test_duplicate_override_keys_deduplicated(self) -> None: + """Duplicate new keys in override list should only appear once.""" + base = {"checks": [{"A": {"p": 1}}]} + override = {"checks": [{"__merge__": True}, {"B": {"p": 2}}, {"B": {"p": 3}}]} + result = deep_merge(base, override) + # B should appear once (first occurrence wins in override_by_key) + b_items = [item for item in result["checks"] if "B" in item] + assert len(b_items) == 1 + + def test_merge_marker_stripped_when_base_key_missing(self) -> None: + """Merge marker should not leak into result when base key doesn't exist.""" + base = {"other": "value"} + override = {"checks": [{"__merge__": True}, {"A": {"p": 1}}]} + result = deep_merge(base, override) + # Marker should be stripped, only A remains + assert result == {"other": "value", "checks": [{"A": {"p": 1}}]} + + +class TestLayeredConfigs: + """Integration tests for layered template + provider configs.""" + + CONFIGS_DIR = Path(__file__).parent.parent / "configs" + + def test_template_has_no_commands(self) -> None: + """Templates should define validations only, no commands.""" + template = merge_yaml_files([self.CONFIGS_DIR / "templates" / "kaas.yaml"]) + assert "commands" not in template, "Template should not contain commands" + assert "tests" in template, "Template must contain tests" + assert "validations" in template["tests"], "Template must contain validations" + + def test_layered_merge_has_both(self) -> None: + """Template + provider merge should have both commands and tests.""" + merged = merge_yaml_files([ + self.CONFIGS_DIR / "templates" / "kaas.yaml", + self.CONFIGS_DIR / "aws" / "eks-layered.yaml", + ]) + assert "commands" in merged, "Merged config must have commands from provider" + assert "tests" in merged, "Merged config must have tests from template" + assert "kubernetes" in merged["commands"], "Commands must have kubernetes key" + steps = merged["commands"]["kubernetes"]["steps"] + assert any(s["name"] == "provision_cluster" for s in steps) + + def test_context_overrides_flow_through(self) -> None: + """Provider context values should appear in the merged config.""" + merged = merge_yaml_files([ + self.CONFIGS_DIR / "templates" / "kaas.yaml", + self.CONFIGS_DIR / "aws" / "eks-layered.yaml", + ]) + assert merged.get("context", {}).get("total_gpus") == "1" + + def test_standalone_eks_still_works(self) -> None: + """Self-contained eks.yaml should parse with both commands and tests.""" + standalone = merge_yaml_files([self.CONFIGS_DIR / "aws" / "eks.yaml"]) + assert "commands" in standalone + assert "tests" in standalone + assert "validations" in standalone["tests"] + + def test_layered_checks_match_standalone_structure(self) -> None: + """Layered and standalone should have the same validation check names.""" + standalone = merge_yaml_files([self.CONFIGS_DIR / "aws" / "eks.yaml"]) + layered = merge_yaml_files([ + self.CONFIGS_DIR / "templates" / "kaas.yaml", + self.CONFIGS_DIR / "aws" / "eks-layered.yaml", + ]) + + def get_check_names(config: dict[str, Any]) -> set[str]: + names: set[str] = set() + for group in config.get("tests", {}).get("validations", {}).values(): + for check in group: + if isinstance(check, dict): + names.update(check.keys()) + return names + + standalone_checks = get_check_names(standalone) + layered_checks = get_check_names(layered) + assert standalone_checks == layered_checks, ( + f"Check mismatch: standalone-only={standalone_checks - layered_checks}, " + f"layered-only={layered_checks - standalone_checks}" + ) + + def test_all_templates_are_validation_only(self) -> None: + """All template YAML files should have tests but no commands.""" + template_dir = self.CONFIGS_DIR / "templates" + for yaml_file in sorted(template_dir.glob("*.yaml")): + config = merge_yaml_files([yaml_file]) + assert "commands" not in config, f"{yaml_file.name} should not have commands" + assert "tests" in config, f"{yaml_file.name} must have tests" diff --git a/uv.lock b/uv.lock index cfc33b62..eb565962 100644 --- a/uv.lock +++ b/uv.lock @@ -661,7 +661,7 @@ requires-dist = [ { name = "pytest", specifier = ">=8.3.4,<9.0.0" }, { name = "pyyaml", specifier = ">=6.0.2,<7.0.0" }, { name = "reframe-hpc", specifier = ">=4.8.4" }, - { name = "urllib3", specifier = ">=2.6.0,<3.0.0" }, + { name = "urllib3", specifier = ">=2.6.3,<3.0.0" }, ] [[package]] From 3ae5fbd7f8407ac4a2d6f40359d2205bcb48d742 Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Fri, 6 Mar 2026 16:00:40 +0100 Subject: [PATCH 5/8] feat: add Carbide control-plane provider implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the control-plane template for the Carbide provider using carbidecli. Maps template concepts to Carbide resources: - API health → tenant get, site list - Access keys → SSH key group + SSH key CRUD - Tenants → VPC CRUD Includes: - carbide/control-plane.yaml: layered provider config - stubs/carbide/common/carbide.py: shared helper (run_carbide, state mgmt) - stubs/carbide/control-plane/: 10 stub scripts matching template steps Usage: isvctl test run \ -f isvctl/configs/templates/control-plane.yaml \ -f isvctl/configs/carbide/control-plane.yaml Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Fabien Dupont --- isvctl/configs/carbide/control-plane.yaml | 122 ++++++++++++++++++ .../configs/stubs/carbide/common/__init__.py | 9 ++ .../configs/stubs/carbide/common/carbide.py | 78 +++++++++++ .../stubs/carbide/control-plane/check_api.py | 84 ++++++++++++ .../control-plane/create_access_key.py | 94 ++++++++++++++ .../carbide/control-plane/create_tenant.py | 93 +++++++++++++ .../control-plane/delete_access_key.py | 74 +++++++++++ .../carbide/control-plane/delete_tenant.py | 71 ++++++++++ .../control-plane/disable_access_key.py | 67 ++++++++++ .../stubs/carbide/control-plane/get_tenant.py | 69 ++++++++++ .../carbide/control-plane/list_tenants.py | 81 ++++++++++++ .../carbide/control-plane/test_access_key.py | 87 +++++++++++++ .../control-plane/verify_key_rejected.py | 82 ++++++++++++ 13 files changed, 1011 insertions(+) create mode 100644 isvctl/configs/carbide/control-plane.yaml create mode 100644 isvctl/configs/stubs/carbide/common/__init__.py create mode 100644 isvctl/configs/stubs/carbide/common/carbide.py create mode 100644 isvctl/configs/stubs/carbide/control-plane/check_api.py create mode 100644 isvctl/configs/stubs/carbide/control-plane/create_access_key.py create mode 100644 isvctl/configs/stubs/carbide/control-plane/create_tenant.py create mode 100644 isvctl/configs/stubs/carbide/control-plane/delete_access_key.py create mode 100644 isvctl/configs/stubs/carbide/control-plane/delete_tenant.py create mode 100644 isvctl/configs/stubs/carbide/control-plane/disable_access_key.py create mode 100644 isvctl/configs/stubs/carbide/control-plane/get_tenant.py create mode 100644 isvctl/configs/stubs/carbide/control-plane/list_tenants.py create mode 100644 isvctl/configs/stubs/carbide/control-plane/test_access_key.py create mode 100644 isvctl/configs/stubs/carbide/control-plane/verify_key_rejected.py diff --git a/isvctl/configs/carbide/control-plane.yaml b/isvctl/configs/carbide/control-plane.yaml new file mode 100644 index 00000000..c6b292f0 --- /dev/null +++ b/isvctl/configs/carbide/control-plane.yaml @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# Carbide Control Plane - Layered Provider Configuration +# +# Supplies commands (carbidecli stubs) for the control-plane template. +# Tests API health, SSH key lifecycle, and VPC lifecycle. +# +# Mapping to template concepts: +# API health → carbidecli tenant get, site list +# Access keys → SSH key group + SSH key CRUD +# Tenants → VPC CRUD (primary tenant-scoped resource) +# +# Usage: +# isvctl test run \ +# -f isvctl/configs/templates/control-plane.yaml \ +# -f isvctl/configs/carbide/control-plane.yaml +# +# Environment: +# carbidecli handles auth via config or env vars: +# CARBIDE_TOKEN / CARBIDE_API_KEY — Authentication +# CARBIDE_ORG — Organization +# CARBIDE_SITE_ID — Site UUID (required for VPC creation) + +version: "1.0" + +commands: + control_plane: + phases: ["setup", "test", "teardown"] + steps: + # ─── SETUP: API Health ───────────────────────────────────────── + - name: check_api + phase: setup + command: "python3 ../stubs/carbide/control-plane/check_api.py" + timeout: 120 + + # ─── SETUP: Create Resources ─────────────────────────────────── + - name: create_access_key + phase: setup + command: "python3 ../stubs/carbide/control-plane/create_access_key.py" + timeout: 120 + + - name: create_tenant + phase: setup + command: "python3 ../stubs/carbide/control-plane/create_tenant.py" + args: ["--site-id", "{{ context.site_id }}"] + timeout: 120 + + # ─── TEST: Access Key (SSH Key) Lifecycle ────────────────────── + - name: test_access_key + phase: test + command: "python3 ../stubs/carbide/control-plane/test_access_key.py" + args: + - "--access-key-id" + - "{{ steps.create_access_key.access_key_id }}" + timeout: 60 + + - name: disable_access_key + phase: test + command: "python3 ../stubs/carbide/control-plane/disable_access_key.py" + args: + - "--access-key-id" + - "{{ steps.create_access_key.access_key_id }}" + timeout: 60 + + - name: verify_key_rejected + phase: test + command: "python3 ../stubs/carbide/control-plane/verify_key_rejected.py" + args: + - "--access-key-id" + - "{{ steps.create_access_key.access_key_id }}" + timeout: 60 + + # ─── TEST: Tenant (VPC) Lifecycle ────────────────────────────── + - name: list_tenants + phase: test + command: "python3 ../stubs/carbide/control-plane/list_tenants.py" + args: + - "--target-group" + - "{{ steps.create_tenant.tenant_name }}" + timeout: 60 + + - name: get_tenant + phase: test + command: "python3 ../stubs/carbide/control-plane/get_tenant.py" + args: + - "--group-name" + - "{{ steps.create_tenant.tenant_name }}" + timeout: 60 + + # ─── TEARDOWN: Cleanup ───────────────────────────────────────── + - name: delete_access_key + phase: teardown + command: "python3 ../stubs/carbide/control-plane/delete_access_key.py" + args: + - "--username" + - "{{ steps.create_access_key.username }}" + timeout: 60 + output_schema: teardown + + - name: delete_tenant + phase: teardown + command: "python3 ../stubs/carbide/control-plane/delete_tenant.py" + args: + - "--group-name" + - "{{ steps.create_tenant.tenant_name }}" + timeout: 60 + output_schema: teardown + +# Provider-specific context +context: + site_id: "{{ env.CARBIDE_SITE_ID | default('') }}" + +tests: + description: "Carbide control plane: API health, SSH key lifecycle, VPC lifecycle" diff --git a/isvctl/configs/stubs/carbide/common/__init__.py b/isvctl/configs/stubs/carbide/common/__init__.py new file mode 100644 index 00000000..442bd7fe --- /dev/null +++ b/isvctl/configs/stubs/carbide/common/__init__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. diff --git a/isvctl/configs/stubs/carbide/common/carbide.py b/isvctl/configs/stubs/carbide/common/carbide.py new file mode 100644 index 00000000..f892f5cb --- /dev/null +++ b/isvctl/configs/stubs/carbide/common/carbide.py @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Shared helpers for Carbide CLI (carbidecli) stub scripts. + +Provides: + - run_carbide(): Execute carbidecli commands with JSON output parsing + - timed_call(): Same as run_carbide but also returns latency + - load_state() / save_state(): Persist data between steps via JSON file + +Environment: + carbidecli handles authentication via its own config (~/.carbide/config.yaml) + or environment variables (CARBIDE_TOKEN, CARBIDE_API_KEY, CARBIDE_ORG, etc.). +""" + +import json +import os +import subprocess +import time +from pathlib import Path +from typing import Any + + +DEFAULT_STATE_FILE = "/tmp/ncp-carbide-state.json" + + +def run_carbide(*args: str, timeout: int = 120) -> dict[str, Any]: + """Run a carbidecli command and return parsed JSON output. + + Args: + *args: Command arguments (e.g., "tenant", "get") + timeout: Command timeout in seconds + + Returns: + Parsed JSON output from carbidecli + + Raises: + RuntimeError: If the command fails or returns non-JSON output + """ + cmd = ["carbidecli", "-o", "json"] + list(args) + result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + + if result.returncode != 0: + raise RuntimeError(f"carbidecli {' '.join(args)} failed: {result.stderr.strip()}") + + try: + return json.loads(result.stdout) + except json.JSONDecodeError: + raise RuntimeError(f"carbidecli returned non-JSON output: {result.stdout[:500]}") + + +def timed_call(*args: str, timeout: int = 120) -> tuple[dict[str, Any], float]: + """Run a carbidecli command and return (result, latency_seconds).""" + start = time.monotonic() + data = run_carbide(*args, timeout=timeout) + elapsed = time.monotonic() - start + return data, elapsed + + +def load_state(state_file: str | None = None) -> dict[str, Any]: + """Load persisted state from a JSON file.""" + path = Path(state_file or os.environ.get("CARBIDE_STATE_FILE", DEFAULT_STATE_FILE)) + if path.exists(): + return json.loads(path.read_text()) + return {} + + +def save_state(state: dict[str, Any], state_file: str | None = None) -> None: + """Save state to a JSON file for use by subsequent steps.""" + path = Path(state_file or os.environ.get("CARBIDE_STATE_FILE", DEFAULT_STATE_FILE)) + path.write_text(json.dumps(state, indent=2)) diff --git a/isvctl/configs/stubs/carbide/control-plane/check_api.py b/isvctl/configs/stubs/carbide/control-plane/check_api.py new file mode 100644 index 00000000..ca92c5b5 --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/check_api.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Check Carbide API connectivity and health. + +Verifies the Carbide control plane is reachable by running +``carbidecli tenant get`` and ``carbidecli site list``. + +Usage: + python check_api.py --region us-west-2 + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "account_id": "", + "tests": { + "tenant": {"passed": true, "latency_ms": 123}, + "sites": {"passed": true, "latency_ms": 89} + } +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import timed_call + + +def main() -> int: + parser = argparse.ArgumentParser(description="Check Carbide API health") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--services", default="tenant,sites", help="Comma-separated checks") + args = parser.parse_args() + + result: dict[str, Any] = { + "success": False, + "platform": "control_plane", + "tests": {}, + } + + try: + # Test tenant get + tenant_data, tenant_latency = timed_call("tenant", "get") + result["tests"]["tenant"] = { + "passed": True, + "latency_ms": round(tenant_latency * 1000, 2), + } + # Extract tenant/account ID from response + result["account_id"] = tenant_data.get("id", tenant_data.get("tenant_id", "")) + + # Test site list + sites_data, sites_latency = timed_call("site", "list") + result["tests"]["sites"] = { + "passed": True, + "latency_ms": round(sites_latency * 1000, 2), + } + + passed = sum(1 for t in result["tests"].values() if t.get("passed", False)) + total = len(result["tests"]) + result["summary"] = f"{passed}/{total} checks passed" + result["success"] = True + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/control-plane/create_access_key.py b/isvctl/configs/stubs/carbide/control-plane/create_access_key.py new file mode 100644 index 00000000..06ce45b9 --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/create_access_key.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Create SSH key group and SSH key in Carbide. + +Maps the template's "access key" concept to Carbide's SSH key model: +an SSH key group is created first, then an SSH key within it. + +Usage: + python create_access_key.py --username ncp-validation + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "username": "ncp-validation", + "user_id": "", + "access_key_id": "", + "secret_access_key": "" +} +""" + +import argparse +import json +import os +import sys +import time +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide, save_state + + +def main() -> int: + parser = argparse.ArgumentParser(description="Create Carbide SSH key group and key") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--username-prefix", default="ncp-validation") + args = parser.parse_args() + + suffix = int(time.time()) + group_name = f"{args.username_prefix}-group-{suffix}" + key_name = f"{args.username_prefix}-key-{suffix}" + + result: dict[str, Any] = { + "success": False, + "platform": "control_plane", + "username": args.username_prefix, + } + + try: + # Create SSH key group + group_resp = run_carbide("ssh-key-group", "create", "--name", group_name) + group_id = group_resp.get("id", group_resp.get("ssh_key_group_id", "")) + result["user_id"] = group_id + + # Create SSH key within the group + key_resp = run_carbide( + "ssh-key", "create", + "--name", key_name, + "--ssh-key-group-id", group_id, + ) + key_id = key_resp.get("id", key_resp.get("ssh_key_id", "")) + public_key = key_resp.get("public_key", key_resp.get("key", "")) + + result["access_key_id"] = key_id + result["secret_access_key"] = public_key + result["success"] = True + + # Persist IDs for subsequent steps + state = load_state() + state["ssh_key_group_id"] = group_id + state["ssh_key_group_name"] = group_name + state["ssh_key_id"] = key_id + state["ssh_key_name"] = key_name + state["username"] = args.username_prefix + save_state(state) + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/control-plane/create_tenant.py b/isvctl/configs/stubs/carbide/control-plane/create_tenant.py new file mode 100644 index 00000000..1619032c --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/create_tenant.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Create a VPC in Carbide (maps to template's "tenant" concept). + +Requires a site ID, provided via ``--site-id`` or the +``CARBIDE_SITE_ID`` environment variable. + +Usage: + python create_tenant.py --site-id + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "tenant_name": "ncp-vpc-", + "tenant_id": "", + "description": "NCP validation VPC" +} +""" + +import argparse +import json +import os +import sys +import time +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide, save_state + + +def main() -> int: + parser = argparse.ArgumentParser(description="Create Carbide VPC") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--name-prefix", default="ncp-vpc") + parser.add_argument("--site-id", default=os.environ.get("CARBIDE_SITE_ID", "")) + args = parser.parse_args() + + if not args.site_id: + print(json.dumps({ + "success": False, + "platform": "control_plane", + "error": "site-id is required (--site-id or CARBIDE_SITE_ID env var)", + }, indent=2)) + return 1 + + vpc_name = f"{args.name_prefix}-{int(time.time())}" + description = "NCP validation VPC" + + result: dict[str, Any] = { + "success": False, + "platform": "control_plane", + "tenant_name": vpc_name, + } + + try: + resp = run_carbide( + "vpc", "create", + "--name", vpc_name, + "--description", description, + "--site-id", args.site_id, + ) + vpc_id = resp.get("id", resp.get("vpc_id", "")) + + result["tenant_id"] = vpc_id + result["description"] = description + result["success"] = True + + # Persist for subsequent steps + state = load_state() + state["vpc_id"] = vpc_id + state["vpc_name"] = vpc_name + state["site_id"] = args.site_id + save_state(state) + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/control-plane/delete_access_key.py b/isvctl/configs/stubs/carbide/control-plane/delete_access_key.py new file mode 100644 index 00000000..0d644691 --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/delete_access_key.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Delete SSH key group in Carbide (teardown for access key lifecycle). + +The SSH key itself was already deleted by ``disable_access_key``; +this step removes the parent SSH key group. + +Usage: + python delete_access_key.py --username ncp-validation --access-key-id + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "resources_deleted": ["ssh-key-group/"] +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Delete Carbide SSH key group") + parser.add_argument("--username", default="") + parser.add_argument("--access-key-id", default="") + parser.add_argument("--skip-destroy", action="store_true") + args = parser.parse_args() + + result: dict[str, Any] = {"success": False, "platform": "control_plane"} + + if args.skip_destroy: + result["success"] = True + result["skipped"] = True + print(json.dumps(result, indent=2)) + return 0 + + state = load_state() + group_id = state.get("ssh_key_group_id", "") + + try: + run_carbide("ssh-key-group", "delete", "--id", group_id) + result["resources_deleted"] = [f"ssh-key-group/{group_id}"] + result["success"] = True + except RuntimeError as e: + # Already deleted is acceptable + if "not found" in str(e).lower() or "404" in str(e): + result["resources_deleted"] = [f"ssh-key-group/{group_id}"] + result["success"] = True + result["already_deleted"] = True + else: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/control-plane/delete_tenant.py b/isvctl/configs/stubs/carbide/control-plane/delete_tenant.py new file mode 100644 index 00000000..79985ae9 --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/delete_tenant.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Delete VPC in Carbide (teardown for tenant lifecycle). + +Usage: + python delete_tenant.py --group-name ncp-vpc-1234567890 + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "resources_deleted": ["vpc/"] +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Delete Carbide VPC") + parser.add_argument("--group-name", default="", help="VPC name (unused, ID from state)") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--skip-destroy", action="store_true") + args = parser.parse_args() + + result: dict[str, Any] = {"success": False, "platform": "control_plane"} + + if args.skip_destroy: + result["success"] = True + result["skipped"] = True + print(json.dumps(result, indent=2)) + return 0 + + state = load_state() + vpc_id = state.get("vpc_id", "") + + try: + run_carbide("vpc", "delete", "--id", vpc_id) + result["resources_deleted"] = [f"vpc/{vpc_id}"] + result["success"] = True + except RuntimeError as e: + # Already deleted is acceptable + if "not found" in str(e).lower() or "404" in str(e): + result["resources_deleted"] = [f"vpc/{vpc_id}"] + result["success"] = True + result["already_deleted"] = True + else: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/control-plane/disable_access_key.py b/isvctl/configs/stubs/carbide/control-plane/disable_access_key.py new file mode 100644 index 00000000..72fdfbf8 --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/disable_access_key.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Disable (delete) the SSH key in Carbide. + +Carbide does not support disabling SSH keys, so this script deletes +the key instead. The output uses the template's expected field names +(``status: Inactive``) for compatibility. + +Usage: + python disable_access_key.py --username ncp-validation --access-key-id + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "access_key_id": "", + "status": "Inactive" +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Disable (delete) Carbide SSH key") + parser.add_argument("--username", default="") + parser.add_argument("--access-key-id", default="") + args = parser.parse_args() + + state = load_state() + key_id = args.access_key_id or state.get("ssh_key_id", "") + + result: dict[str, Any] = { + "success": False, + "platform": "control_plane", + "access_key_id": key_id, + } + + try: + run_carbide("ssh-key", "delete", "--id", key_id) + result["status"] = "Inactive" + result["success"] = True + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/control-plane/get_tenant.py b/isvctl/configs/stubs/carbide/control-plane/get_tenant.py new file mode 100644 index 00000000..3c33bee6 --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/get_tenant.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Get VPC details from Carbide (maps to template's "get tenant" step). + +Usage: + python get_tenant.py --group-name ncp-vpc-1234567890 + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "tenant_name": "ncp-vpc-1234567890", + "tenant_id": "", + "description": "NCP validation VPC" +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Get Carbide VPC details") + parser.add_argument("--group-name", default="", help="VPC name (or loaded from state)") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + args = parser.parse_args() + + state = load_state() + vpc_id = state.get("vpc_id", "") + vpc_name = args.group_name or state.get("vpc_name", "") + + result: dict[str, Any] = { + "success": False, + "platform": "control_plane", + "tenant_name": vpc_name, + } + + try: + resp = run_carbide("vpc", "get", "--id", vpc_id) + + result["tenant_id"] = resp.get("id", resp.get("vpc_id", "")) + result["tenant_name"] = resp.get("name", vpc_name) + result["description"] = resp.get("description", "NCP validation VPC") + result["success"] = True + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/control-plane/list_tenants.py b/isvctl/configs/stubs/carbide/control-plane/list_tenants.py new file mode 100644 index 00000000..d0fdacd2 --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/list_tenants.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""List VPCs in Carbide (maps to template's "list tenants" step). + +Usage: + python list_tenants.py --target-group ncp-vpc-1234567890 + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "tenants": [{"tenant_name": "...", "tenant_id": "..."}], + "count": 1, + "found_target": true, + "target_tenant": "ncp-vpc-1234567890" +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="List Carbide VPCs") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--target-group", default="", help="VPC name to verify exists") + args = parser.parse_args() + + state = load_state() + target = args.target_group or state.get("vpc_name", "") + + result: dict[str, Any] = { + "success": False, + "platform": "control_plane", + "tenants": [], + } + + try: + resp = run_carbide("vpc", "list") + vpcs = resp if isinstance(resp, list) else resp.get("items", []) + + for vpc in vpcs: + result["tenants"].append({ + "tenant_name": vpc.get("name", ""), + "tenant_id": vpc.get("id", vpc.get("vpc_id", "")), + }) + + result["count"] = len(result["tenants"]) + + if target: + result["target_tenant"] = target + result["found_target"] = any( + t["tenant_name"] == target for t in result["tenants"] + ) + + result["success"] = True + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/control-plane/test_access_key.py b/isvctl/configs/stubs/carbide/control-plane/test_access_key.py new file mode 100644 index 00000000..50e4dcdf --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/test_access_key.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Test that the created SSH key is visible in Carbide. + +Lists SSH keys and verifies the key created by ``create_access_key`` +appears in the listing. + +Usage: + python test_access_key.py --access-key-id + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "authenticated": true, + "identity_id": "", + "account_id": "" +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Test Carbide SSH key visibility") + parser.add_argument("--access-key-id", default="") + parser.add_argument("--secret-access-key", default="") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--wait", type=int, default=0, help="Seconds to wait (unused)") + parser.add_argument("--retries", type=int, default=3, help="Number of retry attempts") + args = parser.parse_args() + + state = load_state() + key_id = args.access_key_id or state.get("ssh_key_id", "") + group_id = state.get("ssh_key_group_id", "") + + result: dict[str, Any] = { + "success": False, + "platform": "control_plane", + "authenticated": False, + } + + try: + # List SSH keys and look for our key + keys_resp = run_carbide("ssh-key", "list") + keys = keys_resp if isinstance(keys_resp, list) else keys_resp.get("items", []) + + found = any( + k.get("id", k.get("ssh_key_id", "")) == key_id + for k in keys + ) + + if found: + result["authenticated"] = True + result["identity_id"] = key_id + # Get tenant/account ID + tenant_resp = run_carbide("tenant", "get") + result["account_id"] = tenant_resp.get("id", tenant_resp.get("tenant_id", "")) + result["success"] = True + else: + result["error"] = f"SSH key {key_id} not found in key listing" + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/stubs/carbide/control-plane/verify_key_rejected.py b/isvctl/configs/stubs/carbide/control-plane/verify_key_rejected.py new file mode 100644 index 00000000..0a02e995 --- /dev/null +++ b/isvctl/configs/stubs/carbide/control-plane/verify_key_rejected.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary + +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +"""Verify the deleted SSH key no longer appears in Carbide. + +Carbide does not have a "reject" concept; instead we verify +that the previously deleted SSH key is absent from the key listing. + +Usage: + python verify_key_rejected.py --access-key-id + +Output JSON: +{ + "success": true, + "platform": "control_plane", + "rejected": true, + "error_code": "KeyNotFound" +} +""" + +import argparse +import json +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from common.carbide import load_state, run_carbide + + +def main() -> int: + parser = argparse.ArgumentParser(description="Verify Carbide SSH key is gone") + parser.add_argument("--access-key-id", default="") + parser.add_argument("--secret-access-key", default="") + parser.add_argument("--region", default=os.environ.get("CARBIDE_REGION", "us-west-2")) + parser.add_argument("--wait", type=int, default=0, help="Seconds to wait (unused)") + parser.add_argument("--retries", type=int, default=3, help="Number of retry attempts") + args = parser.parse_args() + + state = load_state() + key_id = args.access_key_id or state.get("ssh_key_id", "") + + result: dict[str, Any] = { + "success": False, + "platform": "control_plane", + "rejected": False, + } + + try: + keys_resp = run_carbide("ssh-key", "list") + keys = keys_resp if isinstance(keys_resp, list) else keys_resp.get("items", []) + + found = any( + k.get("id", k.get("ssh_key_id", "")) == key_id + for k in keys + ) + + if not found: + result["rejected"] = True + result["error_code"] = "KeyNotFound" + result["success"] = True + else: + result["error"] = f"SSH key {key_id} still present after deletion" + + except Exception as e: + result["error"] = str(e) + + print(json.dumps(result, indent=2)) + # Always exit 0 - let validation check the 'rejected' field + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 38a948bd1f9cf2f3dabb6463f7f400de312d174f Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Fri, 6 Mar 2026 17:38:42 +0100 Subject: [PATCH 6/8] feat: add Carbide network, image-registry, and bare-metal providers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements 3 upstream templates for the Carbide provider: Network (carbide/network.yaml + 8 stubs): VPC CRUD, subnet configuration, VPC isolation, NSG security rules, connectivity and traffic validation via carbidecli. Image Registry (carbide/image-registry.yaml + 6 stubs): OperatingSystem CRUD, instance launch from OS image, install config lifecycle — validates Carbide's image management capabilities. Bare Metal (carbide/bm.yaml + 7 stubs): Instance launch/describe/list/reboot/teardown via carbidecli. Reinstall is skipped (not supported). NIM deploy/teardown reuse the shared template stubs. All providers use the layered approach: isvctl test run -f templates/