diff --git a/Makefile b/Makefile index 073abe54..41e51807 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -MY_ISV_DOMAINS := bare_metal control-plane iam image-registry network observability security vm +MY_ISV_DOMAINS := bare_metal control-plane iam image-registry network observability security storage vm DEMO_TARGETS := $(addprefix demo-,$(MY_ISV_DOMAINS)) .PHONY: help pre-commit build test coverage clean lint format install bump-patch bump-fix bump-minor bump-feat bump-major bump bump-check \ diff --git a/isvctl/configs/providers/aws/config/storage.yaml b/isvctl/configs/providers/aws/config/storage.yaml new file mode 100644 index 00000000..084fa715 --- /dev/null +++ b/isvctl/configs/providers/aws/config/storage.yaml @@ -0,0 +1,173 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# AWS Storage (EBS Block Volume) Validation Configuration +# +# Storage is the block-volume implementation of the storage suite, so +# this config imports the provider-agnostic suites/storage.yaml and supplies +# AWS-specific commands (boto3 + SSH) and settings. The launch_instance / +# teardown steps reuse the VM reference scripts; the volume steps live in +# scripts/storage/. +# +# Steps: +# SETUP +# 1. launch_instance - boto3: provision an instance for the fixture +# 2. create_volume - boto3 + SSH: create/attach/format/mount/seed an EBS volume +# TEST +# 3. snapshot_lifecycle - DATASVC-XX-02 (#321): snapshot -> restore -> verify data +# 4. volume_resize - DATASVC-XX-03 (#322): ModifyVolume + growpart/resize2fs +# 5. volume_persistence - DATASVC-XX-04 (#323): stop/start -> verify reattach + data +# TEARDOWN +# 6. teardown_volume - boto3: detach + delete the fixture volume +# 7. teardown - boto3: terminate the instance + SG + key pair +# +# Usage: +# uv run isvctl test run -f isvctl/configs/providers/aws/config/storage.yaml +# +# Required IAM Permissions: +# ec2:DescribeInstances, ec2:RunInstances, ec2:TerminateInstances, +# ec2:StopInstances, ec2:StartInstances, ec2:DescribeImages, ec2:DescribeSubnets, +# ec2:DescribeVpcs, ec2:CreateKeyPair, ec2:DeleteKeyPair, ec2:CreateSecurityGroup, +# ec2:DeleteSecurityGroup, ec2:AuthorizeSecurityGroupIngress, ec2:CreateTags, +# ec2:CreateVolume, ec2:DeleteVolume, ec2:DescribeVolumes, ec2:AttachVolume, +# ec2:DetachVolume, ec2:ModifyVolume, ec2:DescribeVolumesModifications, +# ec2:CreateSnapshot, ec2:DeleteSnapshot, ec2:DescribeSnapshots + +import: + - ../../../suites/storage.yaml + +version: "1.0" + +commands: + storage: + phases: ["setup", "test", "teardown"] + steps: + # AWS-specific: launch the fixture instance (reuses the VM script) + - name: launch_instance + phase: setup + command: "python3 ../scripts/vm/launch_instance.py" + args: + - "--name" + - "isv-test-block" + - "--instance-type" + - "{{instance_type}}" + - "--region" + - "{{region}}" + timeout: 600 + + # AWS-specific: create + attach + format + mount + seed an EBS volume + - name: create_volume + phase: setup + command: "python3 ../scripts/storage/create_volume.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--key-file" + - "{{steps.launch_instance.key_file}}" + - "--size-gib" + - "{{volume_size_gib}}" + timeout: 600 + + # DATASVC-XX-02: snapshot -> restore -> verify data + - name: snapshot_lifecycle + phase: test + command: "python3 ../scripts/storage/snapshot_lifecycle.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--volume-id" + - "{{steps.create_volume.volume_id}}" + - "--key-file" + - "{{steps.launch_instance.key_file}}" + - "--expected-content" + - "{{steps.create_volume.sentinel_content}}" + timeout: 900 + + # DATASVC-XX-03: ModifyVolume + in-guest growpart/resize2fs + - name: volume_resize + phase: test + command: "python3 ../scripts/storage/volume_resize.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--volume-id" + - "{{steps.create_volume.volume_id}}" + - "--key-file" + - "{{steps.launch_instance.key_file}}" + - "--mount-point" + - "{{steps.create_volume.mount_point}}" + timeout: 900 + + # DATASVC-XX-04: stop/start -> verify reattach + data + # Runs last so the stop/start IP churn does not affect the other tests. + - name: volume_persistence + phase: test + command: "python3 ../scripts/storage/volume_persistence.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--volume-id" + - "{{steps.create_volume.volume_id}}" + - "--key-file" + - "{{steps.launch_instance.key_file}}" + - "--mount-point" + - "{{steps.create_volume.mount_point}}" + - "--expected-content" + - "{{steps.create_volume.sentinel_content}}" + timeout: 900 + + # AWS-specific: detach + delete the fixture volume + - name: teardown_volume + phase: teardown + command: "python3 ../scripts/storage/teardown_volume.py" + args: + - "--region" + - "{{region}}" + - "--volume-id" + - "{{steps.create_volume.volume_id}}" + - "{{teardown_flag}}" + timeout: 300 + + # AWS-specific: terminate the instance (reuses the VM script) + - name: teardown + phase: teardown + command: "python3 ../scripts/vm/teardown.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--delete-key-pair" + - "--delete-security-group" + - "{{teardown_flag}}" + timeout: 600 + +tests: + cluster_name: "aws-storage-validation" + description: "AWS block storage (EBS) validation tests" + + settings: + region: "us-west-2" + instance_type: "m6i.large" + volume_size_gib: "10" + teardown_flag: "{{(env.BLOCK_STORAGE_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" diff --git a/isvctl/configs/providers/aws/scripts/common/ebs.py b/isvctl/configs/providers/aws/scripts/common/ebs.py new file mode 100644 index 00000000..0874b697 --- /dev/null +++ b/isvctl/configs/providers/aws/scripts/common/ebs.py @@ -0,0 +1,328 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared EBS (block volume) helper utilities. + +Centralizes the boto3 volume / snapshot lifecycle used by the +storage validation scripts (create / attach / snapshot / restore / +resize / detach / delete) plus the Nitro NVMe device-path mapping needed +to find an attached volume from inside the guest. + +Device mapping note: on Nitro instances EBS volumes are surfaced as NVMe +devices whose kernel name (``/dev/nvme1n1``) is non-deterministic, but +udev creates a stable by-id symlink that embeds the volume ID with the +dash removed:: + + /dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol0123456789abcdef0 + +Scripts resolve that symlink in-guest (``readlink -f``) rather than +guessing ``/dev/sdf`` vs ``/dev/nvme1n1``. +""" + +from __future__ import annotations + +import sys +import time +from typing import Any + +from botocore.exceptions import BotoCoreError, ClientError + +# Ownership tag so cleanup and audits can tell suite-created volumes apart +# from anything the account already had. +_ISV_CREATED_BY_TAG = {"Key": "CreatedBy", "Value": "isvtest"} + +# AWS error codes meaning the volume/snapshot is already gone - treated as +# success in best-effort cleanup paths. +_NOT_FOUND_CODES = frozenset({"InvalidVolume.NotFound", "InvalidSnapshot.NotFound"}) + + +def nvme_serial_for_volume(volume_id: str) -> str: + """Return the NVMe serial AWS assigns to an EBS volume (the ID minus dashes).""" + return volume_id.replace("-", "") + + +def guest_by_id_path(volume_id: str) -> str: + """Return the stable ``/dev/disk/by-id`` path for an attached EBS volume on Nitro.""" + return f"/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_{nvme_serial_for_volume(volume_id)}" + + +def _tag_spec(resource_type: str, name: str) -> dict[str, Any]: + """Build a TagSpecifications entry carrying the Name and ownership tags.""" + return { + "ResourceType": resource_type, + "Tags": [{"Key": "Name", "Value": name}, _ISV_CREATED_BY_TAG], + } + + +def create_volume( + ec2: Any, + availability_zone: str, + size_gib: int, + *, + volume_type: str = "gp3", + name: str = "isv-validate-block", +) -> str: + """Create an empty EBS volume in ``availability_zone`` and return its ID.""" + response = ec2.create_volume( + AvailabilityZone=availability_zone, + Size=size_gib, + VolumeType=volume_type, + TagSpecifications=[_tag_spec("volume", name)], + ) + return response["VolumeId"] + + +def create_volume_from_snapshot( + ec2: Any, + snapshot_id: str, + availability_zone: str, + *, + volume_type: str = "gp3", + name: str = "isv-validate-restore", +) -> str: + """Create a volume restored from ``snapshot_id`` and return its ID.""" + response = ec2.create_volume( + SnapshotId=snapshot_id, + AvailabilityZone=availability_zone, + VolumeType=volume_type, + TagSpecifications=[_tag_spec("volume", name)], + ) + return response["VolumeId"] + + +def attach_volume(ec2: Any, volume_id: str, instance_id: str, device: str) -> None: + """Attach ``volume_id`` to ``instance_id`` at the requested block device name.""" + ec2.attach_volume(VolumeId=volume_id, InstanceId=instance_id, Device=device) + + +def detach_volume(ec2: Any, volume_id: str, *, force: bool = False) -> None: + """Detach ``volume_id`` from whatever instance it is attached to.""" + ec2.detach_volume(VolumeId=volume_id, Force=force) + + +def delete_volume(ec2: Any, volume_id: str) -> None: + """Delete ``volume_id`` (must already be detached / available).""" + ec2.delete_volume(VolumeId=volume_id) + + +def create_snapshot( + ec2: Any, + volume_id: str, + *, + description: str = "ISV storage validation snapshot", + name: str = "isv-validate-snap", +) -> str: + """Create a point-in-time snapshot of ``volume_id`` and return its ID.""" + response = ec2.create_snapshot( + VolumeId=volume_id, + Description=description, + TagSpecifications=[_tag_spec("snapshot", name)], + ) + return response["SnapshotId"] + + +def delete_snapshot(ec2: Any, snapshot_id: str) -> None: + """Delete ``snapshot_id``.""" + ec2.delete_snapshot(SnapshotId=snapshot_id) + + +def modify_volume_size(ec2: Any, volume_id: str, new_size_gib: int) -> None: + """Request a grow of ``volume_id`` to ``new_size_gib`` via ModifyVolume.""" + ec2.modify_volume(VolumeId=volume_id, Size=new_size_gib) + + +def wait_for_volume_available(ec2: Any, volume_id: str, *, delay: int = 5, max_attempts: int = 60) -> None: + """Block until ``volume_id`` reaches the ``available`` (detached) state.""" + waiter = ec2.get_waiter("volume_available") + waiter.wait(VolumeIds=[volume_id], WaiterConfig={"Delay": delay, "MaxAttempts": max_attempts}) + + +def wait_for_volume_in_use(ec2: Any, volume_id: str, *, delay: int = 5, max_attempts: int = 60) -> None: + """Block until ``volume_id`` reaches the ``in-use`` (attached) state.""" + waiter = ec2.get_waiter("volume_in_use") + waiter.wait(VolumeIds=[volume_id], WaiterConfig={"Delay": delay, "MaxAttempts": max_attempts}) + + +def wait_for_volume_deleted(ec2: Any, volume_id: str, *, delay: int = 5, max_attempts: int = 60) -> None: + """Block until ``volume_id`` is fully deleted.""" + waiter = ec2.get_waiter("volume_deleted") + waiter.wait(VolumeIds=[volume_id], WaiterConfig={"Delay": delay, "MaxAttempts": max_attempts}) + + +def wait_for_snapshot_completed(ec2: Any, snapshot_id: str, *, delay: int = 15, max_attempts: int = 80) -> None: + """Block until ``snapshot_id`` finishes (state ``completed``).""" + waiter = ec2.get_waiter("snapshot_completed") + waiter.wait(SnapshotIds=[snapshot_id], WaiterConfig={"Delay": delay, "MaxAttempts": max_attempts}) + + +def wait_for_modification_complete( + ec2: Any, + volume_id: str, + *, + timeout: int = 900, + interval: int = 15, +) -> str: + """Poll ModifyVolume progress until the new size is usable by the guest. + + A volume modification advances ``modifying -> optimizing -> completed``. + The larger capacity is already visible to the OS once the state reaches + ``optimizing``, so the in-guest grow can proceed without waiting for the + (potentially long) background optimization to finish. + + Args: + ec2: Boto3 EC2 client. + volume_id: Volume being modified. + timeout: Total seconds to wait before giving up. + interval: Seconds between describe calls. + + Returns: + The terminal modification state observed (``optimizing`` or ``completed``). + + Raises: + RuntimeError: If the modification fails or does not reach a usable + state within ``timeout``. + """ + deadline = time.monotonic() + timeout + while True: + response = ec2.describe_volumes_modifications(VolumeIds=[volume_id]) + modifications = response.get("VolumesModifications", []) + state = modifications[0].get("ModificationState") if modifications else None + + if state in ("optimizing", "completed"): + return state + if state == "failed": + status = modifications[0].get("StatusMessage", "") + raise RuntimeError(f"Volume modification failed for {volume_id}: {status}") + + if time.monotonic() >= deadline: + raise RuntimeError(f"Timed out waiting for {volume_id} modification (last state: {state})") + time.sleep(interval) + + +def is_volume_attached_to(ec2: Any, volume_id: str, instance_id: str) -> bool: + """Return True if ``volume_id`` is attached to ``instance_id`` and in-use.""" + response = ec2.describe_volumes(VolumeIds=[volume_id]) + volumes = response.get("Volumes", []) + if not volumes: + return False + volume = volumes[0] + if volume.get("State") != "in-use": + return False + return any(att.get("InstanceId") == instance_id for att in volume.get("Attachments", [])) + + +def detach_and_delete_volume(ec2: Any, volume_id: str) -> str | None: + """Detach (if needed) and delete ``volume_id`` best-effort. + + Returns an error message on failure, or ``None`` if the volume was + deleted or was already gone. Safe to call from ``finally`` blocks. + """ + try: + try: + detach_volume(ec2, volume_id, force=True) + wait_for_volume_available(ec2, volume_id) + except ClientError as e: + code = e.response.get("Error", {}).get("Code", "") + if code in _NOT_FOUND_CODES: + return None + # IncorrectState means it is already detached/available - fall through to delete. + if code != "IncorrectState": + return str(e) + + delete_volume(ec2, volume_id) + wait_for_volume_deleted(ec2, volume_id) + return None + except ClientError as e: + code = e.response.get("Error", {}).get("Code", "") + if code in _NOT_FOUND_CODES: + return None + return str(e) + except BotoCoreError as e: + # Waiter timeouts / transport errors must not propagate out of a + # best-effort cleanup path (callers run this in finally blocks). + return str(e) + + +def delete_snapshot_best_effort(ec2: Any, snapshot_id: str) -> str | None: + """Delete ``snapshot_id`` best-effort. Returns an error message or ``None``.""" + try: + delete_snapshot(ec2, snapshot_id) + return None + except ClientError as e: + code = e.response.get("Error", {}).get("Code", "") + if code in _NOT_FOUND_CODES: + return None + return str(e) + except BotoCoreError as e: + return str(e) + + +def wait_for_attachment_device(host: str, user: str, key_file: str, volume_id: str, *, attempts: int = 30) -> bool: + """Poll over SSH until the volume's by-id symlink exists in the guest. + + Returns True once ``guest_by_id_path(volume_id)`` is present, else False. + """ + # Local import to avoid a hard dependency for callers that only need the + # boto3 helpers (and to keep the common package import-light). + from common.ssh_utils import ssh_run + + by_id = guest_by_id_path(volume_id) + for attempt in range(1, attempts + 1): + rc, _, _ = ssh_run(host, user, key_file, f"test -e {by_id}") + if rc == 0: + return True + print(f" Waiting for {by_id} in guest... (attempt {attempt}/{attempts})", file=sys.stderr) + time.sleep(5) + return False + + +# In-guest: resolve the volume's partition via its stable by-id symlink, mount +# it (idempotently), and print the sentinel file contents to stdout. +SENTINEL_FILENAME = "isv-sentinel.txt" +_READ_SENTINEL_SCRIPT = r""" +set -euo pipefail +BYID="__BYID__" +MOUNT="__MOUNT__" +for _ in $(seq 1 30); do [ -e "__BYID__-part1" ] && break; sleep 2; done +PART=$(readlink -f "__BYID__-part1") +sudo mkdir -p "$MOUNT" +mountpoint -q "$MOUNT" || sudo mount "$PART" "$MOUNT" +cat "$MOUNT/__FILENAME__" +""" + + +def mount_and_read_sentinel( + host: str, + user: str, + key_file: str, + volume_id: str, + mount_point: str, + *, + timeout: int = 120, +) -> tuple[int, str, str]: + """Mount ``volume_id`` in the guest and read the sentinel file. + + Returns ``(exit_code, stdout, stderr)`` from the remote command. ``stdout`` + holds the raw sentinel contents on success (callers byte-compare it). + """ + # Local import keeps the common package import-light for boto3-only callers. + from common.ssh_utils import ssh_run + + script = ( + _READ_SENTINEL_SCRIPT.replace("__BYID__", guest_by_id_path(volume_id)) + .replace("__MOUNT__", mount_point) + .replace("__FILENAME__", SENTINEL_FILENAME) + ) + return ssh_run(host, user, key_file, script, timeout=timeout) diff --git a/isvctl/configs/providers/aws/scripts/storage/create_volume.py b/isvctl/configs/providers/aws/scripts/storage/create_volume.py new file mode 100644 index 00000000..bed254ae --- /dev/null +++ b/isvctl/configs/providers/aws/scripts/storage/create_volume.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Block-storage fixture: create, attach, format, mount, and seed a volume. + +Shared setup step for the storage suite (DATASVC-XX-02/03/04). It +creates an EBS volume in the instance's AZ, attaches it, partitions + +formats it (ext4), mounts it, and writes a sentinel file. The volume ID, +mount point, and sentinel content are passed to the snapshot / resize / +persistence test steps, which all reuse this single fixture. + +Output JSON: +{ + "success": true, + "platform": "storage", + "test_name": "create_volume", + "volume_id": "vol-xxx", + "device": "/dev/sdf", + "mount_point": "/mnt/isv-block", + "size_gib": 10, + "sentinel_path": "/mnt/isv-block/isv-sentinel.txt", + "sentinel_content": "isv-ncp-validate-storage-...", + "operations": { + "create": {"passed": true}, + "attach": {"passed": true}, + "format": {"passed": true}, + "mount": {"passed": true}, + "write_sentinel": {"passed": true} + } +} +""" + +import argparse +import json +import os +import sys +import uuid +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # providers/aws/scripts/ (for common.*) + +import boto3 +from botocore.exceptions import BotoCoreError, ClientError +from common import ebs +from common.ssh_utils import ssh_run, wait_for_ssh + +# Remote setup: resolve the attached device via its stable by-id symlink, +# lay down a single GPT partition, format it ext4, mount it, and write the +# sentinel file. Placeholders are substituted in Python (avoids f-string vs +# shell brace conflicts). +_SETUP_SCRIPT = r""" +set -euo pipefail +BYID="__BYID__" +MOUNT="__MOUNT__" +SENTINEL="$MOUNT/isv-sentinel.txt" +CONTENT="__CONTENT__" +for _ in $(seq 1 30); do [ -e "$BYID" ] && break; sleep 2; done +DEV=$(readlink -f "$BYID") +sudo parted -s "$DEV" mklabel gpt +sudo parted -s "$DEV" mkpart primary ext4 0% 100% +sudo partprobe "$DEV" || true +sudo udevadm settle || true +for _ in $(seq 1 30); do [ -e "__BYID__-part1" ] && break; sleep 2; done +PART=$(readlink -f "__BYID__-part1") +sudo mkfs.ext4 -F "$PART" +sudo mkdir -p "$MOUNT" +sudo mount "$PART" "$MOUNT" +echo -n "$CONTENT" | sudo tee "$SENTINEL" >/dev/null +sudo sync +""" + + +def _render(script: str, **subs: str) -> str: + """Substitute __NAME__ placeholders in a remote shell script template.""" + for key, value in subs.items(): + script = script.replace(f"__{key}__", value) + return script + + +def _fail(op: dict[str, Any], message: str) -> None: + """Mark an operation failed with a message.""" + op["passed"] = False + op["error"] = message + + +def main() -> int: + """Create, attach, format, mount, and seed a block volume; print JSON.""" + parser = argparse.ArgumentParser(description="Block-storage fixture: create + attach + seed a volume") + parser.add_argument("--instance-id", required=True, help="EC2 instance to attach the volume to") + parser.add_argument("--region", default=os.environ.get("AWS_REGION", "us-west-2")) + parser.add_argument("--key-file", required=True, help="Path to SSH private key") + parser.add_argument("--ssh-user", default="ubuntu", help="SSH username") + parser.add_argument("--size-gib", type=int, default=10, help="Volume size in GiB") + parser.add_argument("--device", default="/dev/sdf", help="Requested attach device name") + parser.add_argument("--mount-point", default="/mnt/isv-block", help="In-guest mount point") + args = parser.parse_args() + + sentinel_content = f"isv-ncp-validate-storage-{uuid.uuid4().hex}" + operations: dict[str, dict[str, Any]] = { + "create": {"passed": False}, + "attach": {"passed": False}, + "format": {"passed": False}, + "mount": {"passed": False}, + "write_sentinel": {"passed": False}, + } + result: dict[str, Any] = { + "success": False, + "platform": "storage", + "test_name": "create_volume", + "instance_id": args.instance_id, + "volume_id": None, + "device": args.device, + "mount_point": args.mount_point, + "size_gib": args.size_gib, + "sentinel_path": f"{args.mount_point}/isv-sentinel.txt", + "sentinel_content": sentinel_content, + "operations": operations, + } + + ec2 = boto3.client("ec2", region_name=args.region) + + try: + instances = ec2.describe_instances(InstanceIds=[args.instance_id]) + instance = instances["Reservations"][0]["Instances"][0] + availability_zone = instance["Placement"]["AvailabilityZone"] + public_ip = instance.get("PublicIpAddress") + result["availability_zone"] = availability_zone + + volume_id = ebs.create_volume(ec2, availability_zone, args.size_gib, name="isv-validate-block") + result["volume_id"] = volume_id + ebs.wait_for_volume_available(ec2, volume_id) + operations["create"]["passed"] = True + + ebs.attach_volume(ec2, volume_id, args.instance_id, args.device) + ebs.wait_for_volume_in_use(ec2, volume_id) + operations["attach"]["passed"] = True + except (ClientError, BotoCoreError) as e: + result["error"] = f"Volume create/attach failed: {e}" + print(json.dumps(result, indent=2)) + return 1 + + if not public_ip: + result["error"] = "Instance has no public IP for SSH" + print(json.dumps(result, indent=2)) + return 1 + + if not wait_for_ssh(public_ip, args.ssh_user, args.key_file, max_attempts=30, interval=10): + result["error"] = "SSH not ready on fixture instance" + print(json.dumps(result, indent=2)) + return 1 + + if not ebs.wait_for_attachment_device(public_ip, args.ssh_user, args.key_file, volume_id): + _fail(operations["format"], "Attached volume did not appear in guest") + result["error"] = "Attached volume device never appeared in guest" + print(json.dumps(result, indent=2)) + return 1 + + setup = _render( + _SETUP_SCRIPT, + BYID=ebs.guest_by_id_path(volume_id), + MOUNT=args.mount_point, + CONTENT=sentinel_content, + ) + rc, _, err = ssh_run(public_ip, args.ssh_user, args.key_file, setup, timeout=180) + if rc == 0: + operations["format"]["passed"] = True + operations["mount"]["passed"] = True + operations["write_sentinel"]["passed"] = True + else: + _fail(operations["format"], f"Guest format/mount/seed failed (rc={rc}): {err.strip()[:300]}") + result["error"] = "Guest format/mount/seed failed" + + result["success"] = all(op["passed"] for op in operations.values()) + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/aws/scripts/storage/snapshot_lifecycle.py b/isvctl/configs/providers/aws/scripts/storage/snapshot_lifecycle.py new file mode 100644 index 00000000..ded557ed --- /dev/null +++ b/isvctl/configs/providers/aws/scripts/storage/snapshot_lifecycle.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DATASVC-XX-02: verify volume snapshots (issue #321). + +Snapshots the fixture volume, restores the snapshot to a brand-new volume, +attaches + mounts that restore on the same instance, and byte-compares the +sentinel file against the content the fixture wrote. A successful round-trip +proves the snapshot captured the data and the restore is independently +usable. The restored volume and snapshot are cleaned up in a finally block. + +Output JSON: +{ + "success": true, + "platform": "storage", + "test_name": "snapshot_lifecycle", + "volume_id": "vol-source", + "snapshot_id": "snap-xxx", + "restored_volume_id": "vol-restore", + "operations": { + "create_snapshot": {"passed": true}, + "restore_volume": {"passed": true}, + "verify_data": {"passed": true, "content_matches": true} + } +} +""" + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # providers/aws/scripts/ (for common.*) + +import boto3 +from botocore.exceptions import BotoCoreError, ClientError +from common import ebs +from common.ssh_utils import ssh_run, wait_for_ssh + + +def _fail(op: dict[str, Any], message: str) -> None: + """Mark an operation failed with a message.""" + op["passed"] = False + op["error"] = message + + +def main() -> int: + """Snapshot the fixture volume, restore it, verify the data; print JSON.""" + parser = argparse.ArgumentParser(description="Verify volume snapshots (DATASVC-XX-02)") + parser.add_argument("--instance-id", required=True, help="EC2 instance for the restored volume") + parser.add_argument("--region", default=os.environ.get("AWS_REGION", "us-west-2")) + parser.add_argument("--volume-id", required=True, help="Source (fixture) volume to snapshot") + parser.add_argument("--key-file", required=True, help="Path to SSH private key") + parser.add_argument("--ssh-user", default="ubuntu", help="SSH username") + parser.add_argument("--expected-content", required=True, help="Sentinel content written by the fixture") + parser.add_argument("--restore-device", default="/dev/sdg", help="Attach device for the restored volume") + parser.add_argument("--restore-mount", default="/mnt/isv-restored", help="Mount point for the restored volume") + args = parser.parse_args() + + operations: dict[str, dict[str, Any]] = { + "create_snapshot": {"passed": False}, + "restore_volume": {"passed": False}, + "verify_data": {"passed": False}, + } + result: dict[str, Any] = { + "success": False, + "platform": "storage", + "test_name": "snapshot_lifecycle", + "volume_id": args.volume_id, + "snapshot_id": None, + "restored_volume_id": None, + "operations": operations, + } + + ec2 = boto3.client("ec2", region_name=args.region) + + snapshot_id: str | None = None + restored_volume_id: str | None = None + try: + instances = ec2.describe_instances(InstanceIds=[args.instance_id]) + instance = instances["Reservations"][0]["Instances"][0] + availability_zone = instance["Placement"]["AvailabilityZone"] + public_ip = instance.get("PublicIpAddress") + + # Flush the guest page cache so the snapshot is consistent with the + # sentinel write (best-effort; the fixture already synced after write). + if public_ip: + ssh_run(public_ip, args.ssh_user, args.key_file, "sudo sync") + + try: + snapshot_id = ebs.create_snapshot(ec2, args.volume_id, name="isv-validate-snap") + result["snapshot_id"] = snapshot_id + ebs.wait_for_snapshot_completed(ec2, snapshot_id) + operations["create_snapshot"]["passed"] = True + except (ClientError, BotoCoreError) as e: + _fail(operations["create_snapshot"], str(e)) + result["error"] = f"CreateSnapshot failed: {e}" + # snapshot_id may be set if creation succeeded but the wait failed - + # pass it so a partially-created snapshot is still cleaned up. + return _emit(result, ec2, restored_volume_id, snapshot_id) + + try: + restored_volume_id = ebs.create_volume_from_snapshot(ec2, snapshot_id, availability_zone) + result["restored_volume_id"] = restored_volume_id + ebs.wait_for_volume_available(ec2, restored_volume_id) + ebs.attach_volume(ec2, restored_volume_id, args.instance_id, args.restore_device) + ebs.wait_for_volume_in_use(ec2, restored_volume_id) + operations["restore_volume"]["passed"] = True + except (ClientError, BotoCoreError) as e: + _fail(operations["restore_volume"], str(e)) + result["error"] = f"Restore failed: {e}" + return _emit(result, ec2, restored_volume_id, snapshot_id) + + if not public_ip or not wait_for_ssh(public_ip, args.ssh_user, args.key_file, max_attempts=30, interval=10): + _fail(operations["verify_data"], "SSH not ready for restore verification") + result["error"] = "SSH not ready" + return _emit(result, ec2, restored_volume_id, snapshot_id) + + if not ebs.wait_for_attachment_device(public_ip, args.ssh_user, args.key_file, restored_volume_id): + _fail(operations["verify_data"], "Restored volume did not appear in guest") + return _emit(result, ec2, restored_volume_id, snapshot_id) + + rc, out, err = ebs.mount_and_read_sentinel( + public_ip, args.ssh_user, args.key_file, restored_volume_id, args.restore_mount + ) + if rc != 0: + _fail(operations["verify_data"], f"Could not read sentinel on restore (rc={rc}): {err.strip()[:300]}") + return _emit(result, ec2, restored_volume_id, snapshot_id) + + content_matches = out.strip() == args.expected_content.strip() + operations["verify_data"]["content_matches"] = content_matches + if content_matches: + operations["verify_data"]["passed"] = True + else: + _fail(operations["verify_data"], "Restored sentinel does not match fixture content") + + result["success"] = all(op["passed"] for op in operations.values()) + return _emit(result, ec2, restored_volume_id, snapshot_id) + except (ClientError, BotoCoreError) as e: + result["error"] = str(e) + return _emit(result, ec2, restored_volume_id, snapshot_id) + + +def _emit( + result: dict[str, Any], + ec2: Any = None, + restored_volume_id: str | None = None, + snapshot_id: str | None = None, +) -> int: + """Best-effort cleanup of the restored volume + snapshot, then print JSON.""" + if ec2 is not None and restored_volume_id: + err = ebs.detach_and_delete_volume(ec2, restored_volume_id) + if err: + result.setdefault("cleanup_errors", []).append(err) + if ec2 is not None and snapshot_id: + err = ebs.delete_snapshot_best_effort(ec2, snapshot_id) + if err: + result.setdefault("cleanup_errors", []).append(err) + if result.get("cleanup_errors"): + result["success"] = False + cleanup_msg = f"Cleanup failed: {'; '.join(result['cleanup_errors'])}" + result["error"] = f"{result['error']}; {cleanup_msg}" if result.get("error") else cleanup_msg + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/aws/scripts/storage/teardown_volume.py b/isvctl/configs/providers/aws/scripts/storage/teardown_volume.py new file mode 100644 index 00000000..d91c8c58 --- /dev/null +++ b/isvctl/configs/providers/aws/scripts/storage/teardown_volume.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Detach and delete the storage fixture volume (teardown). + +Best-effort cleanup of the volume created by ``create_volume.py``. Runs +before the instance teardown so the volume is removed explicitly rather +than left dangling. The restored volume and snapshot from the snapshot +test clean themselves up in their own step. + +Output JSON: +{ + "success": true, + "platform": "storage", + "test_name": "teardown_volume", + "resources_deleted": ["vol-xxx"] +} +""" + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # providers/aws/scripts/ (for common.*) + +import boto3 +from common import ebs + + +def main() -> int: + """Detach and delete the fixture volume; print JSON result.""" + parser = argparse.ArgumentParser(description="Teardown storage fixture volume") + parser.add_argument("--region", default=os.environ.get("AWS_REGION", "us-west-2")) + parser.add_argument("--volume-id", required=True, help="Fixture volume to delete") + parser.add_argument("--skip-destroy", action="store_true", help="Skip actual destroy") + args = parser.parse_args() + + result: dict[str, Any] = { + "success": False, + "platform": "storage", + "test_name": "teardown_volume", + "resources_deleted": [], + } + + if args.skip_destroy: + result["success"] = True + result["message"] = f"Volume {args.volume_id} preserved (--skip-destroy); delete manually when done" + print(json.dumps(result, indent=2)) + return 0 + + ec2 = boto3.client("ec2", region_name=args.region) + error = ebs.detach_and_delete_volume(ec2, args.volume_id) + if error: + result["error"] = error + result["message"] = f"Failed to delete volume {args.volume_id}" + else: + result["success"] = True + result["resources_deleted"].append(args.volume_id) + result["message"] = f"Volume {args.volume_id} deleted" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/aws/scripts/storage/volume_persistence.py b/isvctl/configs/providers/aws/scripts/storage/volume_persistence.py new file mode 100644 index 00000000..df366c40 --- /dev/null +++ b/isvctl/configs/providers/aws/scripts/storage/volume_persistence.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DATASVC-XX-04: persistent volumes survive instance restarts (issue #323). + +Stops and starts the instance, then confirms the attached block volume is +still attached after the restart and that its sentinel data is intact. The +volume is re-mounted in the guest (a manual mount does not survive a reboot) +and the sentinel file is byte-compared against the fixture content. + +Output JSON: +{ + "success": true, + "platform": "storage", + "test_name": "volume_persistence", + "volume_id": "vol-xxx", + "operations": { + "stop": {"passed": true}, + "start": {"passed": true}, + "verify_attached": {"passed": true}, + "verify_data": {"passed": true, "content_matches": true} + } +} +""" + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # providers/aws/scripts/ (for common.*) + +import boto3 +from botocore.exceptions import BotoCoreError, ClientError +from common import ebs +from common.ec2 import wait_for_public_ip +from common.ssh_utils import wait_for_ssh + + +def _fail(op: dict[str, Any], message: str) -> None: + """Mark an operation failed with a message.""" + op["passed"] = False + op["error"] = message + + +def main() -> int: + """Stop/start the instance and verify the volume + data survive; print JSON.""" + parser = argparse.ArgumentParser(description="Verify block volume persistence across restart (DATASVC-XX-04)") + parser.add_argument("--instance-id", required=True, help="EC2 instance to restart") + parser.add_argument("--region", default=os.environ.get("AWS_REGION", "us-west-2")) + parser.add_argument("--volume-id", required=True, help="Volume expected to persist (fixture volume)") + parser.add_argument("--key-file", required=True, help="Path to SSH private key") + parser.add_argument("--ssh-user", default="ubuntu", help="SSH username") + parser.add_argument("--mount-point", default="/mnt/isv-block", help="In-guest mount point") + parser.add_argument("--expected-content", required=True, help="Sentinel content written by the fixture") + args = parser.parse_args() + + operations: dict[str, dict[str, Any]] = { + "stop": {"passed": False}, + "start": {"passed": False}, + "verify_attached": {"passed": False}, + "verify_data": {"passed": False}, + } + result: dict[str, Any] = { + "success": False, + "platform": "storage", + "test_name": "volume_persistence", + "volume_id": args.volume_id, + "instance_id": args.instance_id, + "operations": operations, + } + + ec2 = boto3.client("ec2", region_name=args.region) + + try: + ec2.stop_instances(InstanceIds=[args.instance_id]) + ec2.get_waiter("instance_stopped").wait( + InstanceIds=[args.instance_id], WaiterConfig={"Delay": 15, "MaxAttempts": 40} + ) + operations["stop"]["passed"] = True + + ec2.start_instances(InstanceIds=[args.instance_id]) + ec2.get_waiter("instance_status_ok").wait( + InstanceIds=[args.instance_id], WaiterConfig={"Delay": 15, "MaxAttempts": 40} + ) + instances = ec2.describe_instances(InstanceIds=[args.instance_id]) + instance = instances["Reservations"][0]["Instances"][0] + public_ip = instance.get("PublicIpAddress") or wait_for_public_ip(ec2, args.instance_id) + if not public_ip: + _fail(operations["start"], "No public IP after restart") + result["error"] = "No public IP after restart" + print(json.dumps(result, indent=2)) + return 1 + if not wait_for_ssh(public_ip, args.ssh_user, args.key_file, max_attempts=40, interval=15): + _fail(operations["start"], "SSH not ready after restart") + result["error"] = "SSH not ready after restart" + print(json.dumps(result, indent=2)) + return 1 + operations["start"]["passed"] = True + except (ClientError, BotoCoreError) as e: + result["error"] = f"Restart failed: {e}" + print(json.dumps(result, indent=2)) + return 1 + + if ebs.is_volume_attached_to(ec2, args.volume_id, args.instance_id): + operations["verify_attached"]["passed"] = True + else: + _fail(operations["verify_attached"], "Volume not attached after restart") + + if operations["verify_attached"]["passed"]: + rc, out, err = ebs.mount_and_read_sentinel( + public_ip, args.ssh_user, args.key_file, args.volume_id, args.mount_point + ) + if rc != 0: + _fail(operations["verify_data"], f"Could not read sentinel after restart (rc={rc}): {err.strip()[:300]}") + else: + content_matches = out.strip() == args.expected_content.strip() + operations["verify_data"]["content_matches"] = content_matches + if content_matches: + operations["verify_data"]["passed"] = True + else: + _fail(operations["verify_data"], "Sentinel content changed across restart") + + result["success"] = all(op["passed"] for op in operations.values()) + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/aws/scripts/storage/volume_resize.py b/isvctl/configs/providers/aws/scripts/storage/volume_resize.py new file mode 100644 index 00000000..4d325604 --- /dev/null +++ b/isvctl/configs/providers/aws/scripts/storage/volume_resize.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DATASVC-XX-03: verify volume resizing (issue #322). + +Grows the fixture volume with ModifyVolume, then grows the in-guest +partition (growpart) and filesystem (resize2fs) and confirms the larger +capacity is visible to the guest. Online resize keeps the volume mounted +throughout, so no data is lost. + +Output JSON: +{ + "success": true, + "platform": "storage", + "test_name": "volume_resize", + "volume_id": "vol-xxx", + "old_size_gib": 10, + "new_size_gib": 15, + "fs_bytes_before": 10434441216, + "fs_bytes_after": 15728623616, + "operations": { + "modify_volume": {"passed": true}, + "grow_partition": {"passed": true}, + "resize_filesystem": {"passed": true}, + "verify_size": {"passed": true} + } +} +""" + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # providers/aws/scripts/ (for common.*) + +import boto3 +from botocore.exceptions import BotoCoreError, ClientError +from common import ebs +from common.ssh_utils import ssh_run, wait_for_ssh + +_READ_FS_BYTES = r""" +set -euo pipefail +MOUNT="__MOUNT__" +df -B1 --output=size "$MOUNT" | tail -1 | tr -d ' ' +""" + +_GROWPART = r""" +set -euo pipefail +DEV=$(readlink -f "__BYID__") +sudo growpart "$DEV" 1 +sudo partprobe "$DEV" || true +sudo udevadm settle || true +""" + +_RESIZE2FS = r""" +set -euo pipefail +PART=$(readlink -f "__BYID__-part1") +sudo resize2fs "$PART" +""" + + +def _fail(op: dict[str, Any], message: str) -> None: + """Mark an operation failed with a message.""" + op["passed"] = False + op["error"] = message + + +def main() -> int: + """Grow the fixture volume + in-guest filesystem and verify; print JSON.""" + parser = argparse.ArgumentParser(description="Verify volume resizing (DATASVC-XX-03)") + parser.add_argument("--instance-id", required=True, help="EC2 instance the volume is attached to") + parser.add_argument("--region", default=os.environ.get("AWS_REGION", "us-west-2")) + parser.add_argument("--volume-id", required=True, help="Volume to resize (fixture volume)") + parser.add_argument("--key-file", required=True, help="Path to SSH private key") + parser.add_argument("--ssh-user", default="ubuntu", help="SSH username") + parser.add_argument("--mount-point", default="/mnt/isv-block", help="In-guest mount point") + parser.add_argument("--grow-gib", type=int, default=5, help="GiB to add to the volume") + args = parser.parse_args() + + operations: dict[str, dict[str, Any]] = { + "modify_volume": {"passed": False}, + "grow_partition": {"passed": False}, + "resize_filesystem": {"passed": False}, + "verify_size": {"passed": False}, + } + result: dict[str, Any] = { + "success": False, + "platform": "storage", + "test_name": "volume_resize", + "volume_id": args.volume_id, + "operations": operations, + } + + ec2 = boto3.client("ec2", region_name=args.region) + by_id = ebs.guest_by_id_path(args.volume_id) + + try: + instances = ec2.describe_instances(InstanceIds=[args.instance_id]) + public_ip = instances["Reservations"][0]["Instances"][0].get("PublicIpAddress") + + volumes = ec2.describe_volumes(VolumeIds=[args.volume_id]) + old_size = volumes["Volumes"][0]["Size"] + new_size = old_size + args.grow_gib + result["old_size_gib"] = old_size + result["new_size_gib"] = new_size + + try: + ebs.modify_volume_size(ec2, args.volume_id, new_size) + ebs.wait_for_modification_complete(ec2, args.volume_id) + operations["modify_volume"]["passed"] = True + except (ClientError, BotoCoreError, RuntimeError) as e: + _fail(operations["modify_volume"], str(e)) + result["error"] = f"ModifyVolume failed: {e}" + print(json.dumps(result, indent=2)) + return 1 + except (ClientError, BotoCoreError) as e: + result["error"] = str(e) + print(json.dumps(result, indent=2)) + return 1 + + if not public_ip or not wait_for_ssh(public_ip, args.ssh_user, args.key_file, max_attempts=30, interval=10): + result["error"] = "SSH not ready for in-guest resize" + print(json.dumps(result, indent=2)) + return 1 + + rc, before_out, _ = ssh_run( + public_ip, args.ssh_user, args.key_file, _READ_FS_BYTES.replace("__MOUNT__", args.mount_point) + ) + fs_before = int(before_out.strip()) if rc == 0 and before_out.strip().isdigit() else None + result["fs_bytes_before"] = fs_before + + rc, _, err = ssh_run(public_ip, args.ssh_user, args.key_file, _GROWPART.replace("__BYID__", by_id)) + if rc == 0: + operations["grow_partition"]["passed"] = True + else: + _fail(operations["grow_partition"], f"growpart failed (rc={rc}): {err.strip()[:300]}") + + if operations["grow_partition"]["passed"]: + rc, _, err = ssh_run(public_ip, args.ssh_user, args.key_file, _RESIZE2FS.replace("__BYID__", by_id)) + if rc == 0: + operations["resize_filesystem"]["passed"] = True + else: + _fail(operations["resize_filesystem"], f"resize2fs failed (rc={rc}): {err.strip()[:300]}") + + rc, after_out, _ = ssh_run( + public_ip, args.ssh_user, args.key_file, _READ_FS_BYTES.replace("__MOUNT__", args.mount_point) + ) + fs_after = int(after_out.strip()) if rc == 0 and after_out.strip().isdigit() else None + result["fs_bytes_after"] = fs_after + + ebs_grew = ec2.describe_volumes(VolumeIds=[args.volume_id])["Volumes"][0]["Size"] == result["new_size_gib"] + fs_grew = fs_before is not None and fs_after is not None and fs_after > fs_before + if ebs_grew and fs_grew: + operations["verify_size"]["passed"] = True + else: + _fail( + operations["verify_size"], + f"Size did not grow as expected (ebs_grew={ebs_grew}, fs_before={fs_before}, fs_after={fs_after})", + ) + + result["success"] = all(op["passed"] for op in operations.values()) + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/my-isv/config/storage.yaml b/isvctl/configs/providers/my-isv/config/storage.yaml new file mode 100644 index 00000000..af1d5fa3 --- /dev/null +++ b/isvctl/configs/providers/my-isv/config/storage.yaml @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# my-isv Storage Validation Configuration - Living Example +# +# Storage is the block-volume implementation of the storage suite, so +# this config imports the provider-agnostic suites/storage.yaml and wires the +# commands to the generic template scripts. Those scripts ship with +# dummy-success values so this example runs end-to-end without any real cloud +# infrastructure (ISVCTL_DEMO_MODE=1, used by `make demo-test`). +# +# The launch_instance / teardown steps reuse the VM scaffold scripts; the +# volume steps live in scripts/storage/. +# +# Steps: +# SETUP +# 1. launch_instance - Provision the fixture instance +# 2. create_volume - Create + attach + format + mount + seed a volume +# TEST +# 3. snapshot_lifecycle - DATASVC-XX-02 (#321): snapshot -> restore -> verify data +# 4. volume_resize - DATASVC-XX-03 (#322): grow volume + growpart/resize2fs +# 5. volume_persistence - DATASVC-XX-04 (#323): stop/start -> verify reattach + data +# TEARDOWN +# 6. teardown_volume - Detach + delete the fixture volume +# 7. teardown - Terminate the instance +# +# Usage: +# uv run isvctl test run -f isvctl/configs/providers/my-isv/config/storage.yaml + +import: + - ../../../suites/storage.yaml + +version: "1.0" + +commands: + storage: + phases: ["setup", "test", "teardown"] + steps: + - name: launch_instance + phase: setup + command: "python ../scripts/vm/launch_instance.py" + args: + - "--name" + - "isv-test-block" + - "--instance-type" + - "{{instance_type}}" + - "--region" + - "{{region}}" + timeout: 60 + + - name: create_volume + phase: setup + command: "python ../scripts/storage/create_volume.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--key-file" + - "{{steps.launch_instance.key_file}}" + - "--size-gib" + - "{{volume_size_gib}}" + timeout: 60 + + - name: snapshot_lifecycle + phase: test + command: "python ../scripts/storage/snapshot_lifecycle.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--volume-id" + - "{{steps.create_volume.volume_id}}" + - "--key-file" + - "{{steps.launch_instance.key_file}}" + - "--expected-content" + - "{{steps.create_volume.sentinel_content}}" + timeout: 60 + + - name: volume_resize + phase: test + command: "python ../scripts/storage/volume_resize.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--volume-id" + - "{{steps.create_volume.volume_id}}" + - "--key-file" + - "{{steps.launch_instance.key_file}}" + - "--mount-point" + - "{{steps.create_volume.mount_point}}" + timeout: 60 + + - name: volume_persistence + phase: test + command: "python ../scripts/storage/volume_persistence.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--volume-id" + - "{{steps.create_volume.volume_id}}" + - "--key-file" + - "{{steps.launch_instance.key_file}}" + - "--mount-point" + - "{{steps.create_volume.mount_point}}" + - "--expected-content" + - "{{steps.create_volume.sentinel_content}}" + timeout: 60 + + - name: teardown_volume + phase: teardown + command: "python ../scripts/storage/teardown_volume.py" + args: + - "--region" + - "{{region}}" + - "--volume-id" + - "{{steps.create_volume.volume_id}}" + - "{{teardown_flag}}" + timeout: 60 + + - name: teardown + phase: teardown + command: "python ../scripts/vm/teardown.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--delete-key-pair" + - "--delete-security-group" + timeout: 60 + +tests: + cluster_name: "my-isv-storage-validation" + description: "my-isv block storage validation (generic stubs, dummy overrides)" + + settings: + region: "my-isv-region-1" + instance_type: "my-isv.standard.1x" + volume_size_gib: "10" + teardown_flag: "{{(env.BLOCK_STORAGE_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" diff --git a/isvctl/configs/providers/my-isv/scripts/README.md b/isvctl/configs/providers/my-isv/scripts/README.md index 54bdecc1..87a92367 100644 --- a/isvctl/configs/providers/my-isv/scripts/README.md +++ b/isvctl/configs/providers/my-isv/scripts/README.md @@ -31,6 +31,7 @@ template, then fill in the TODOs. | `control-plane/` | 10 | [`suites/control-plane.yaml`](../../../suites/control-plane.yaml) | [`config/control-plane.yaml`](../config/control-plane.yaml) | [`providers/aws/scripts/control-plane/`](../../aws/scripts/control-plane/) | | `vm/` | 9 | [`suites/vm.yaml`](../../../suites/vm.yaml) | [`config/vm.yaml`](../config/vm.yaml) | [`providers/aws/scripts/vm/`](../../aws/scripts/vm/) | | `bare_metal/` | 12 | [`suites/bare_metal.yaml`](../../../suites/bare_metal.yaml) | [`config/bare_metal.yaml`](../config/bare_metal.yaml) | [`providers/aws/scripts/bare_metal/`](../../aws/scripts/bare_metal/) | +| `storage/` | 5 | [`suites/storage.yaml`](../../../suites/storage.yaml) | [`config/storage.yaml`](../config/storage.yaml) | [`providers/aws/scripts/storage/`](../../aws/scripts/storage/) | | `network/` | 18 | [`suites/network.yaml`](../../../suites/network.yaml) | [`config/network.yaml`](../config/network.yaml) | [`providers/aws/scripts/network/`](../../aws/scripts/network/) | | `observability/` | 1 | [`suites/observability.yaml`](../../../suites/observability.yaml) | [`config/observability.yaml`](../config/observability.yaml) | [`providers/aws/scripts/observability/`](../../aws/scripts/observability/) | | `image-registry/` | 7 | [`suites/image-registry.yaml`](../../../suites/image-registry.yaml) | [`config/image-registry.yaml`](../config/image-registry.yaml) | [`providers/aws/scripts/image-registry/`](../../aws/scripts/image-registry/) | diff --git a/isvctl/configs/providers/my-isv/scripts/storage/create_volume.py b/isvctl/configs/providers/my-isv/scripts/storage/create_volume.py new file mode 100644 index 00000000..e04b96d6 --- /dev/null +++ b/isvctl/configs/providers/my-isv/scripts/storage/create_volume.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Block-storage fixture: create, attach, format, mount, and seed a volume. + +Provider-agnostic template - replace the TODO block with your platform's +block-volume API calls plus in-guest formatting. This single fixture is +shared by the snapshot, resize, and persistence tests, so the volume must +be left attached, formatted, mounted, and seeded with the sentinel file. + +Required JSON output fields: + success (bool) - true iff every operation passed + platform (str) - "storage" + test_name (str) - "create_volume" + volume_id (str) - identifier of the created volume (consumed by later steps) + mount_point (str) - where the volume is mounted in the guest + sentinel_content (str) - exact content written to the sentinel file (consumed by later steps) + operations: { + "create": {"passed": bool, "error": str?}, + "attach": {"passed": bool, "error": str?}, + "format": {"passed": bool, "error": str?}, + "mount": {"passed": bool, "error": str?}, + "write_sentinel": {"passed": bool, "error": str?} + } + +Usage: + python create_volume.py --instance-id --region --key-file --size-gib 10 + +Reference implementation (AWS): + ../aws/storage/create_volume.py +""" + +import argparse +import json +import os +import sys +import uuid +from typing import Any + +# ISVCTL_DEMO_MODE=1 enables demo-success output (used by `make demo-test`). +DEMO_MODE = os.environ.get("ISVCTL_DEMO_MODE") == "1" + + +def main() -> int: + """Create + attach + format + mount + seed a block volume; print JSON.""" + parser = argparse.ArgumentParser(description="Block-storage fixture: create + seed a volume") + parser.add_argument("--instance-id", default="", help="Instance to attach the volume to") + parser.add_argument("--region", default="", help="Cloud region") + parser.add_argument("--key-file", default="", help="Path to SSH private key") + parser.add_argument("--size-gib", type=int, default=10, help="Volume size in GiB") + parser.add_argument("--mount-point", default="/mnt/isv-block", help="In-guest mount point") + args = parser.parse_args() + + sentinel_content = f"isv-ncp-validate-storage-{uuid.uuid4().hex}" + operations: dict[str, dict[str, Any]] = { + "create": {"passed": False}, + "attach": {"passed": False}, + "format": {"passed": False}, + "mount": {"passed": False}, + "write_sentinel": {"passed": False}, + } + result: dict[str, Any] = { + "success": False, + "platform": "storage", + "test_name": "create_volume", + "volume_id": "", + "mount_point": args.mount_point, + "size_gib": args.size_gib, + "sentinel_content": sentinel_content, + "operations": operations, + } + + # ╔══════════════════════════════════════════════════════════════════╗ + # ║ TODO: Replace this block with your platform's implementation ║ + # ║ ║ + # ║ 1. CreateVolume(size_gib) in the instance's zone ║ + # ║ -> operations["create"]["passed"] = True ║ + # ║ 2. AttachVolume(volume_id, instance_id) ║ + # ║ -> operations["attach"]["passed"] = True ║ + # ║ 3. Over SSH: partition + mkfs the attached device ║ + # ║ -> operations["format"]["passed"] = True ║ + # ║ 4. Over SSH: mount it at args.mount_point ║ + # ║ -> operations["mount"]["passed"] = True ║ + # ║ 5. Over SSH: write sentinel_content to /isv-sentinel.txt ║ + # ║ -> operations["write_sentinel"]["passed"] = True ║ + # ║ 6. result["volume_id"] = volume_id ║ + # ║ 7. result["success"] = all(op["passed"] for op in operations…) ║ + # ╚══════════════════════════════════════════════════════════════════╝ + + if DEMO_MODE: + result["volume_id"] = f"dummy-vol-{uuid.uuid4().hex[:8]}" + for op in operations.values(): + op["passed"] = True + result["success"] = True + else: + operations["create"]["error"] = "Not implemented" + result["error"] = "Not implemented - replace with your platform's create/attach/format/mount logic" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/my-isv/scripts/storage/snapshot_lifecycle.py b/isvctl/configs/providers/my-isv/scripts/storage/snapshot_lifecycle.py new file mode 100644 index 00000000..06ede89f --- /dev/null +++ b/isvctl/configs/providers/my-isv/scripts/storage/snapshot_lifecycle.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DATASVC-XX-02: verify volume snapshots (issue #321). + +Provider-agnostic template - replace the TODO block with your platform's +snapshot + restore API calls. Snapshot the fixture volume, restore it to a +new volume, attach + mount that restore, and byte-compare the sentinel file +against the content the fixture wrote. Clean up the restored volume and the +snapshot afterward. + +Required JSON output fields: + success (bool) - true iff every operation passed + platform (str) - "storage" + test_name (str) - "snapshot_lifecycle" + volume_id (str) - source (fixture) volume that was snapshotted + snapshot_id (str) - identifier of the created snapshot + operations: { + "create_snapshot": {"passed": bool, "error": str?}, + "restore_volume": {"passed": bool, "error": str?}, + "verify_data": {"passed": bool, "content_matches": bool, "error": str?} + } + +Usage: + python snapshot_lifecycle.py --volume-id --expected-content + +Reference implementation (AWS): + ../aws/storage/snapshot_lifecycle.py +""" + +import argparse +import json +import os +import sys +import uuid +from typing import Any + +# ISVCTL_DEMO_MODE=1 enables demo-success output (used by `make demo-test`). +DEMO_MODE = os.environ.get("ISVCTL_DEMO_MODE") == "1" + + +def main() -> int: + """Snapshot, restore, and verify a volume; print JSON result.""" + parser = argparse.ArgumentParser(description="Verify volume snapshots (DATASVC-XX-02)") + parser.add_argument("--instance-id", default="", help="Instance for the restored volume") + parser.add_argument("--region", default="", help="Cloud region") + parser.add_argument("--volume-id", default="", help="Source (fixture) volume to snapshot") + parser.add_argument("--key-file", default="", help="Path to SSH private key") + parser.add_argument("--expected-content", default="", help="Sentinel content written by the fixture") + args = parser.parse_args() + + operations: dict[str, dict[str, Any]] = { + "create_snapshot": {"passed": False}, + "restore_volume": {"passed": False}, + "verify_data": {"passed": False}, + } + result: dict[str, Any] = { + "success": False, + "platform": "storage", + "test_name": "snapshot_lifecycle", + "volume_id": args.volume_id, + "snapshot_id": "", + "operations": operations, + } + + # ╔══════════════════════════════════════════════════════════════════╗ + # ║ TODO: Replace this block with your platform's implementation ║ + # ║ ║ + # ║ 1. CreateSnapshot(volume_id); wait until complete ║ + # ║ -> operations["create_snapshot"]["passed"] = True ║ + # ║ 2. CreateVolume(from snapshot); attach to the instance ║ + # ║ -> operations["restore_volume"]["passed"] = True ║ + # ║ 3. Over SSH: mount the restore, read /isv-sentinel.txt ║ + # ║ matches = (content == args.expected_content) ║ + # ║ -> operations["verify_data"]["content_matches"] = matches ║ + # ║ -> operations["verify_data"]["passed"] = matches ║ + # ║ 4. Best-effort: delete the restored volume and the snapshot ║ + # ║ 5. result["success"] = all(op["passed"] for op in operations…) ║ + # ╚══════════════════════════════════════════════════════════════════╝ + + if DEMO_MODE: + result["snapshot_id"] = f"dummy-snap-{uuid.uuid4().hex[:8]}" + result["restored_volume_id"] = f"dummy-vol-{uuid.uuid4().hex[:8]}" + operations["create_snapshot"]["passed"] = True + operations["restore_volume"]["passed"] = True + operations["verify_data"]["passed"] = True + operations["verify_data"]["content_matches"] = True + result["success"] = True + else: + operations["create_snapshot"]["error"] = "Not implemented" + result["error"] = "Not implemented - replace with your platform's snapshot/restore logic" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/my-isv/scripts/storage/teardown_volume.py b/isvctl/configs/providers/my-isv/scripts/storage/teardown_volume.py new file mode 100644 index 00000000..4633e746 --- /dev/null +++ b/isvctl/configs/providers/my-isv/scripts/storage/teardown_volume.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Detach and delete the storage fixture volume (teardown). + +Provider-agnostic template - replace the TODO block with your platform's +detach + delete API calls for the fixture volume created by +``create_volume.py``. + +Required JSON output fields: + success (bool) - whether cleanup succeeded + platform (str) - "storage" + test_name (str) - "teardown_volume" + resources_deleted (list) - identifiers of volumes that were deleted + message (str) - human-readable summary + +Usage: + python teardown_volume.py --volume-id --region + +Reference implementation (AWS): + ../aws/storage/teardown_volume.py +""" + +import argparse +import json +import os +import sys +from typing import Any + +# ISVCTL_DEMO_MODE=1 enables demo-success output (used by `make demo-test`). +DEMO_MODE = os.environ.get("ISVCTL_DEMO_MODE") == "1" + + +def main() -> int: + """Detach and delete the fixture volume; print JSON result.""" + parser = argparse.ArgumentParser(description="Teardown storage fixture volume") + parser.add_argument("--region", default="", help="Cloud region") + parser.add_argument("--volume-id", default="", help="Fixture volume to delete") + parser.add_argument("--skip-destroy", action="store_true", help="Skip actual destroy") + args = parser.parse_args() + + result: dict[str, Any] = { + "success": False, + "platform": "storage", + "test_name": "teardown_volume", + "resources_deleted": [], + "message": "", + } + + if args.skip_destroy: + result["success"] = True + result["message"] = f"Volume {args.volume_id} preserved (--skip-destroy)" + print(json.dumps(result, indent=2)) + return 0 + + # ╔══════════════════════════════════════════════════════════════════╗ + # ║ TODO: Replace this block with your platform's implementation ║ + # ║ ║ + # ║ 1. Detach the volume from the instance (if still attached) ║ + # ║ 2. Delete the volume ║ + # ║ result["resources_deleted"].append(args.volume_id) ║ + # ║ 3. result["success"] = True ║ + # ╚══════════════════════════════════════════════════════════════════╝ + + if DEMO_MODE: + if args.volume_id: + result["resources_deleted"].append(args.volume_id) + result["message"] = "Fixture volume deleted" + result["success"] = True + else: + result["error"] = "Not implemented - replace with your platform's detach + delete logic" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/my-isv/scripts/storage/volume_persistence.py b/isvctl/configs/providers/my-isv/scripts/storage/volume_persistence.py new file mode 100644 index 00000000..d760c3d4 --- /dev/null +++ b/isvctl/configs/providers/my-isv/scripts/storage/volume_persistence.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DATASVC-XX-04: persistent volumes survive instance restarts (issue #323). + +Provider-agnostic template - replace the TODO block with your platform's +stop/start API calls. Stop and start the instance, then confirm the block +volume is still attached and its sentinel data is intact (the volume must be +re-mounted after the restart). + +Required JSON output fields: + success (bool) - true iff every operation passed + platform (str) - "storage" + test_name (str) - "volume_persistence" + volume_id (str) - volume expected to persist + operations: { + "stop": {"passed": bool, "error": str?}, + "start": {"passed": bool, "error": str?}, + "verify_attached": {"passed": bool, "error": str?}, + "verify_data": {"passed": bool, "content_matches": bool, "error": str?} + } + +Usage: + python volume_persistence.py --instance-id --volume-id --expected-content + +Reference implementation (AWS): + ../aws/storage/volume_persistence.py +""" + +import argparse +import json +import os +import sys +from typing import Any + +# ISVCTL_DEMO_MODE=1 enables demo-success output (used by `make demo-test`). +DEMO_MODE = os.environ.get("ISVCTL_DEMO_MODE") == "1" + + +def main() -> int: + """Stop/start the instance and verify the volume + data survive; print JSON.""" + parser = argparse.ArgumentParser(description="Verify block volume persistence across restart (DATASVC-XX-04)") + parser.add_argument("--instance-id", default="", help="Instance to restart") + parser.add_argument("--region", default="", help="Cloud region") + parser.add_argument("--volume-id", default="", help="Volume expected to persist") + parser.add_argument("--key-file", default="", help="Path to SSH private key") + parser.add_argument("--mount-point", default="/mnt/isv-block", help="In-guest mount point") + parser.add_argument("--expected-content", default="", help="Sentinel content written by the fixture") + args = parser.parse_args() + + operations: dict[str, dict[str, Any]] = { + "stop": {"passed": False}, + "start": {"passed": False}, + "verify_attached": {"passed": False}, + "verify_data": {"passed": False}, + } + result: dict[str, Any] = { + "success": False, + "platform": "storage", + "test_name": "volume_persistence", + "volume_id": args.volume_id, + "instance_id": args.instance_id, + "operations": operations, + } + + # ╔══════════════════════════════════════════════════════════════════╗ + # ║ TODO: Replace this block with your platform's implementation ║ + # ║ ║ + # ║ 1. Stop the instance; wait for it to reach stopped ║ + # ║ -> operations["stop"]["passed"] = True ║ + # ║ 2. Start the instance; wait for it to be reachable over SSH ║ + # ║ -> operations["start"]["passed"] = True ║ + # ║ 3. Confirm the volume is still attached to the instance ║ + # ║ -> operations["verify_attached"]["passed"] = True ║ + # ║ 4. Over SSH: re-mount the volume, read /isv-sentinel.txt ║ + # ║ matches = (content == args.expected_content) ║ + # ║ -> operations["verify_data"]["content_matches"] = matches ║ + # ║ -> operations["verify_data"]["passed"] = matches ║ + # ║ 5. result["success"] = all(op["passed"] for op in operations…) ║ + # ╚══════════════════════════════════════════════════════════════════╝ + + if DEMO_MODE: + operations["stop"]["passed"] = True + operations["start"]["passed"] = True + operations["verify_attached"]["passed"] = True + operations["verify_data"]["passed"] = True + operations["verify_data"]["content_matches"] = True + result["success"] = True + else: + operations["stop"]["error"] = "Not implemented" + result["error"] = "Not implemented - replace with your platform's stop/start + verify logic" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/my-isv/scripts/storage/volume_resize.py b/isvctl/configs/providers/my-isv/scripts/storage/volume_resize.py new file mode 100644 index 00000000..65675f16 --- /dev/null +++ b/isvctl/configs/providers/my-isv/scripts/storage/volume_resize.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DATASVC-XX-03: verify volume resizing (issue #322). + +Provider-agnostic template - replace the TODO block with your platform's +volume-grow API call plus the in-guest partition + filesystem grow. Confirm +the larger capacity is visible to the guest after the resize. + +Required JSON output fields: + success (bool) - true iff every operation passed + platform (str) - "storage" + test_name (str) - "volume_resize" + volume_id (str) - volume that was resized + operations: { + "modify_volume": {"passed": bool, "error": str?}, + "grow_partition": {"passed": bool, "error": str?}, + "resize_filesystem": {"passed": bool, "error": str?}, + "verify_size": {"passed": bool, "error": str?} + } + +Usage: + python volume_resize.py --volume-id --mount-point /mnt/isv-block + +Reference implementation (AWS): + ../aws/storage/volume_resize.py +""" + +import argparse +import json +import os +import sys +from typing import Any + +# ISVCTL_DEMO_MODE=1 enables demo-success output (used by `make demo-test`). +DEMO_MODE = os.environ.get("ISVCTL_DEMO_MODE") == "1" + + +def main() -> int: + """Grow the volume + in-guest filesystem and verify; print JSON result.""" + parser = argparse.ArgumentParser(description="Verify volume resizing (DATASVC-XX-03)") + parser.add_argument("--instance-id", default="", help="Instance the volume is attached to") + parser.add_argument("--region", default="", help="Cloud region") + parser.add_argument("--volume-id", default="", help="Volume to resize (fixture volume)") + parser.add_argument("--key-file", default="", help="Path to SSH private key") + parser.add_argument("--mount-point", default="/mnt/isv-block", help="In-guest mount point") + parser.add_argument("--grow-gib", type=int, default=5, help="GiB to add to the volume") + args = parser.parse_args() + + operations: dict[str, dict[str, Any]] = { + "modify_volume": {"passed": False}, + "grow_partition": {"passed": False}, + "resize_filesystem": {"passed": False}, + "verify_size": {"passed": False}, + } + result: dict[str, Any] = { + "success": False, + "platform": "storage", + "test_name": "volume_resize", + "volume_id": args.volume_id, + "operations": operations, + } + + # ╔══════════════════════════════════════════════════════════════════╗ + # ║ TODO: Replace this block with your platform's implementation ║ + # ║ ║ + # ║ 1. Read the current volume size, then grow it by args.grow_gib ║ + # ║ -> operations["modify_volume"]["passed"] = True ║ + # ║ 2. Over SSH: growpart the partition ║ + # ║ -> operations["grow_partition"]["passed"] = True ║ + # ║ 3. Over SSH: resize2fs (or equivalent) the filesystem ║ + # ║ -> operations["resize_filesystem"]["passed"] = True ║ + # ║ 4. Confirm the guest sees the larger filesystem size ║ + # ║ -> operations["verify_size"]["passed"] = True ║ + # ║ 5. result["success"] = all(op["passed"] for op in operations…) ║ + # ╚══════════════════════════════════════════════════════════════════╝ + + if DEMO_MODE: + result["old_size_gib"] = 10 + result["new_size_gib"] = 10 + args.grow_gib + for op in operations.values(): + op["passed"] = True + result["success"] = True + else: + operations["modify_volume"]["error"] = "Not implemented" + result["error"] = "Not implemented - replace with your platform's resize + growpart/resize2fs logic" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index 8cd8bded..a77d77f6 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -14,6 +14,7 @@ Suites: [`network`](network.yaml), [`vm`](vm.yaml), [`bare_metal`](bare_metal.yaml), +[`storage`](storage.yaml), [`observability`](observability.yaml), [`k8s`](k8s.yaml), [`slurm`](slurm.yaml), @@ -108,6 +109,24 @@ For the domain / script-count / AWS-reference overview see the | `teardown` | teardown | `providers/my-isv/scripts/bare_metal/teardown.py` | `resources_deleted`, `message` | | `verify_teardown` | teardown | `providers/my-isv/scripts/bare_metal/verify_terminated.py` | `checks.instance_terminated`, `checks.sg_deleted` | +### Storage (`storage.yaml`) + +Umbrella suite for the storage capability area. Today it covers persistent block +storage (DATASVC-XX-02/03/04); future object/file storage checks land here too rather +than spawning new suites. A shared fixture (`launch_instance` + `create_volume`) +provisions one instance with a single attached, formatted, mounted, and seeded block +volume. The three test-phase steps all reuse that fixture. + +| Step | Phase | Script | Key JSON Fields | +|------|-------|--------|-----------------| +| `launch_instance` | setup | `providers/my-isv/scripts/vm/launch_instance.py` | `instance_id`, `state`, `public_ip`, `key_file` (reuses VM script) | +| `create_volume` | setup | `providers/my-isv/scripts/storage/create_volume.py` | `volume_id`, `mount_point`, `sentinel_content`, `operations.{create,attach,format,mount,write_sentinel}` | +| `snapshot_lifecycle` | test | `providers/my-isv/scripts/storage/snapshot_lifecycle.py` | `volume_id`, `snapshot_id`, `operations.{create_snapshot,restore_volume,verify_data}` (verify_data includes `content_matches`) | +| `volume_resize` | test | `providers/my-isv/scripts/storage/volume_resize.py` | `volume_id`, `operations.{modify_volume,grow_partition,resize_filesystem,verify_size}` | +| `volume_persistence` | test | `providers/my-isv/scripts/storage/volume_persistence.py` | `volume_id`, `operations.{stop,start,verify_attached,verify_data}` (verify_data includes `content_matches`) | +| `teardown_volume` | teardown | `providers/my-isv/scripts/storage/teardown_volume.py` | `resources_deleted`, `message` | +| `teardown` | teardown | `providers/my-isv/scripts/vm/teardown.py` | `resources_deleted`, `message` (reuses VM script) | + ### Kubernetes (`k8s.yaml`) | Step | Phase | Script | diff --git a/isvctl/configs/suites/storage.yaml b/isvctl/configs/suites/storage.yaml new file mode 100644 index 00000000..476229d1 --- /dev/null +++ b/isvctl/configs/suites/storage.yaml @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Storage Validation - Canonical Contract +# +# Validations-only contract for the storage capability area. Provider +# configs import this file and supply their own commands (steps + scripts). +# +# This file defines WHAT to validate, not HOW to run it. Each validation +# only inspects JSON output fields. As long as your scripts output the +# right fields, the validations pass regardless of which cloud you use. +# +# Scope: today this suite covers persistent block storage (EBS-style +# volumes); it is the umbrella home for future storage checks (object, +# file, ...) so they don't each spawn a new suite file. +# +# Coverage notes: +# - A shared fixture launches one instance and creates + attaches a single +# block volume (formatted, mounted, seeded with a sentinel file). All +# three test-phase checks reuse that fixture: +# - snapshot_lifecycle - DATASVC-XX-02 (issue #321): snapshot the volume, +# restore it to a new volume, attach + mount the restore, and verify the +# sentinel data survived the round-trip. +# - volume_resize - DATASVC-XX-03 (issue #322): grow the volume +# (ModifyVolume), then grow the in-guest partition + filesystem and +# confirm the larger size is visible to the guest. +# - volume_persistence - DATASVC-XX-04 (issue #323): stop + start the +# instance and confirm the volume stays attached with its data intact. +# +# Quick start: +# 1. Copy providers/my-isv/config/storage.yaml to providers//config/storage.yaml +# 2. Replace stub paths with your platform scripts +# 3. Run: uv run isvctl test run -f isvctl/configs/providers//config/storage.yaml +# +# See also: +# - my-isv living example: providers/my-isv/config/storage.yaml +# - AWS reference: providers/aws/config/storage.yaml + providers/aws/scripts/storage/ +# - Per-step field breakdown: ../suites/README.md + +version: "1.0" + +tests: + platform: storage + cluster_name: "storage-validation" + description: "Storage: block-volume snapshots, resizing, and persistence across restarts" + + settings: + # Provider-specific - override in your provider config + region: "" + instance_type: "" + volume_size_gib: "10" + teardown_flag: "{{(env.BLOCK_STORAGE_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" + + # ─── Validations ───────────────────────────────────────────────── + # These are PROVIDER-AGNOSTIC. They only check JSON field names/values. + validations: + # ─── Fixture: instance + attached, formatted, seeded volume ────── + instance_launched: + step: launch_instance + checks: + InstanceStateCheck: + expected_state: "running" + + fixture_volume: + step: create_volume + checks: + StepSuccessCheck: {} + FieldExistsCheck: + fields: ["volume_id", "mount_point", "operations"] + CrudOperationsCheck: + operations: ["create", "attach", "format", "mount", "write_sentinel"] + + # ─── DATASVC-XX-02: volume snapshots (issue #321) ──────────────── + snapshot_lifecycle: + step: snapshot_lifecycle + checks: + StepSuccessCheck: {} + FieldExistsCheck: + fields: ["volume_id", "snapshot_id", "operations"] + CrudOperationsCheck: + operations: ["create_snapshot", "restore_volume", "verify_data"] + + # ─── DATASVC-XX-03: volume resizing (issue #322) ───────────────── + volume_resize: + step: volume_resize + checks: + StepSuccessCheck: {} + FieldExistsCheck: + fields: ["volume_id", "operations"] + CrudOperationsCheck: + operations: ["modify_volume", "grow_partition", "resize_filesystem", "verify_size"] + + # ─── DATASVC-XX-04: persistence across restarts (issue #323) ───── + volume_persistence: + step: volume_persistence + checks: + StepSuccessCheck: {} + FieldExistsCheck: + fields: ["volume_id", "operations"] + CrudOperationsCheck: + operations: ["stop", "start", "verify_attached", "verify_data"] + + # ─── Teardown ──────────────────────────────────────────────────── + teardown_checks: + step: teardown_volume + checks: + StepSuccessCheck: {} + + exclude: + labels: [] diff --git a/isvctl/tests/test_aws_storage_scripts.py b/isvctl/tests/test_aws_storage_scripts.py new file mode 100644 index 00000000..8fc24e7e --- /dev/null +++ b/isvctl/tests/test_aws_storage_scripts.py @@ -0,0 +1,508 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for AWS storage reference scripts (DATASVC-XX-02/03/04).""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest +from botocore.exceptions import ClientError + +ISVCTL_ROOT = Path(__file__).resolve().parents[1] +AWS_STORAGE_SCRIPTS = ISVCTL_ROOT / "configs" / "providers" / "aws" / "scripts" / "storage" + +EXPECTED_CONTENT = "isv-ncp-validate-storage-deadbeef" + + +def _load_script(script_name: str) -> ModuleType: + """Load an AWS storage script as a module for direct testing.""" + script_path = AWS_STORAGE_SCRIPTS / script_name + spec = importlib.util.spec_from_file_location(f"test_storage_{script_path.stem}", script_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _client_error(code: str) -> ClientError: + """Build a ClientError with the given AWS error code.""" + return ClientError({"Error": {"Code": code, "Message": code}}, "Op") + + +class FakeWaiter: + """No-op boto3 waiter.""" + + def wait(self, **kwargs: Any) -> None: + """Return immediately - nothing to wait for in tests.""" + + +class FakeEc2: + """Minimal fake EC2 client covering the raw calls the scripts make.""" + + def __init__( + self, + *, + availability_zone: str = "us-west-2a", + public_ip: str | None = "203.0.113.10", + volume_size: int = 10, + attached_instance: str = "i-fixture", + ) -> None: + """Seed instance/volume describe responses used by the scripts.""" + self.availability_zone = availability_zone + self.public_ip = public_ip + self.size = volume_size + self.attached_instance = attached_instance + self.stopped: list[str] = [] + self.started: list[str] = [] + + def describe_instances(self, InstanceIds: list[str]) -> dict[str, Any]: + """Return a single instance with AZ + public IP.""" + return { + "Reservations": [ + { + "Instances": [ + { + "Placement": {"AvailabilityZone": self.availability_zone}, + "PublicIpAddress": self.public_ip, + } + ] + } + ] + } + + def describe_volumes(self, VolumeIds: list[str]) -> dict[str, Any]: + """Return the (mutable) size and an in-use attachment.""" + return { + "Volumes": [ + { + "Size": self.size, + "State": "in-use", + "Attachments": [{"InstanceId": self.attached_instance}], + } + ] + } + + def modify_volume(self, VolumeId: str, Size: int) -> dict[str, Any]: + """Apply the new size so a later describe reflects the grow.""" + self.size = Size + return {"VolumeModification": {"ModificationState": "modifying"}} + + def stop_instances(self, InstanceIds: list[str]) -> None: + """Record the stop call.""" + self.stopped.extend(InstanceIds) + + def start_instances(self, InstanceIds: list[str]) -> None: + """Record the start call.""" + self.started.extend(InstanceIds) + + def get_waiter(self, name: str) -> FakeWaiter: + """Return a no-op waiter.""" + return FakeWaiter() + + +def _patch_ebs_volume_ops(monkeypatch: pytest.MonkeyPatch, module: ModuleType) -> None: + """Patch the ebs boto3 wrappers to fast, side-effect-free fakes.""" + monkeypatch.setattr(module.ebs, "create_volume", lambda *a, **k: "vol-fixture") + monkeypatch.setattr(module.ebs, "create_volume_from_snapshot", lambda *a, **k: "vol-restore") + monkeypatch.setattr(module.ebs, "attach_volume", lambda *a, **k: None) + monkeypatch.setattr(module.ebs, "wait_for_volume_available", lambda *a, **k: None) + monkeypatch.setattr(module.ebs, "wait_for_volume_in_use", lambda *a, **k: None) + monkeypatch.setattr(module.ebs, "wait_for_attachment_device", lambda *a, **k: True) + + +# --------------------------------------------------------------------------- +# common/ebs.py pure helpers +# --------------------------------------------------------------------------- + + +def test_nvme_serial_and_by_id_path_drop_dash() -> None: + """The Nitro NVMe serial / by-id path embed the volume ID minus the dash.""" + module = _load_script("create_volume.py") + assert module.ebs.nvme_serial_for_volume("vol-0123456789abcdef0") == "vol0123456789abcdef0" + assert module.ebs.guest_by_id_path("vol-0123456789abcdef0") == ( + "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol0123456789abcdef0" + ) + + +def test_detach_and_delete_volume_treats_missing_as_gone() -> None: + """A volume that is already gone is a successful cleanup (returns None).""" + module = _load_script("teardown_volume.py") + + class GoneEc2: + def detach_volume(self, **kwargs: Any) -> None: + raise _client_error("InvalidVolume.NotFound") + + assert module.ebs.detach_and_delete_volume(GoneEc2(), "vol-x") is None + + +def test_wait_for_modification_complete_returns_usable_state() -> None: + """The modification wait returns once the new size is usable (optimizing).""" + module = _load_script("volume_resize.py") + + class ModEc2: + def describe_volumes_modifications(self, VolumeIds: list[str]) -> dict[str, Any]: + return {"VolumesModifications": [{"ModificationState": "optimizing"}]} + + assert module.ebs.wait_for_modification_complete(ModEc2(), "vol-x") == "optimizing" + + +def test_wait_for_modification_complete_raises_on_failure() -> None: + """A failed modification raises rather than looping until timeout.""" + module = _load_script("volume_resize.py") + + class FailedEc2: + def describe_volumes_modifications(self, VolumeIds: list[str]) -> dict[str, Any]: + return {"VolumesModifications": [{"ModificationState": "failed", "StatusMessage": "nope"}]} + + with pytest.raises(RuntimeError, match="modification failed"): + module.ebs.wait_for_modification_complete(FailedEc2(), "vol-x") + + +# --------------------------------------------------------------------------- +# create_volume.py +# --------------------------------------------------------------------------- + + +def test_create_volume_happy_path(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + """A clean create/attach/format/mount/seed flow passes every operation.""" + module = _load_script("create_volume.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2()) + _patch_ebs_volume_ops(monkeypatch, module) + monkeypatch.setattr(module, "wait_for_ssh", lambda *a, **k: True) + monkeypatch.setattr(module, "ssh_run", lambda *a, **k: (0, "", "")) + monkeypatch.setattr(sys, "argv", ["create_volume.py", "--instance-id", "i-fixture", "--key-file", "/tmp/k.pem"]) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert payload["success"] is True + assert payload["volume_id"] == "vol-fixture" + assert all(op["passed"] for op in payload["operations"].values()) + + +def test_create_volume_guest_setup_failure_fails_step( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A non-zero in-guest setup makes the fixture step fail.""" + module = _load_script("create_volume.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2()) + _patch_ebs_volume_ops(monkeypatch, module) + monkeypatch.setattr(module, "wait_for_ssh", lambda *a, **k: True) + monkeypatch.setattr(module, "ssh_run", lambda *a, **k: (1, "", "mkfs failed")) + monkeypatch.setattr(sys, "argv", ["create_volume.py", "--instance-id", "i-fixture", "--key-file", "/tmp/k.pem"]) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 1 + assert payload["success"] is False + assert payload["operations"]["format"]["passed"] is False + + +# --------------------------------------------------------------------------- +# snapshot_lifecycle.py +# --------------------------------------------------------------------------- + + +def test_snapshot_lifecycle_happy_path(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + """A matching restored sentinel passes the snapshot round-trip.""" + module = _load_script("snapshot_lifecycle.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2()) + _patch_ebs_volume_ops(monkeypatch, module) + monkeypatch.setattr(module.ebs, "create_snapshot", lambda *a, **k: "snap-1") + monkeypatch.setattr(module.ebs, "wait_for_snapshot_completed", lambda *a, **k: None) + monkeypatch.setattr(module.ebs, "mount_and_read_sentinel", lambda *a, **k: (0, EXPECTED_CONTENT, "")) + monkeypatch.setattr(module.ebs, "detach_and_delete_volume", lambda *a, **k: None) + monkeypatch.setattr(module.ebs, "delete_snapshot_best_effort", lambda *a, **k: None) + monkeypatch.setattr(module, "wait_for_ssh", lambda *a, **k: True) + monkeypatch.setattr(module, "ssh_run", lambda *a, **k: (0, "", "")) + monkeypatch.setattr( + sys, + "argv", + [ + "snapshot_lifecycle.py", + "--instance-id", + "i-fixture", + "--volume-id", + "vol-fixture", + "--key-file", + "/tmp/k.pem", + "--expected-content", + EXPECTED_CONTENT, + ], + ) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert payload["success"] is True + assert payload["snapshot_id"] == "snap-1" + assert payload["operations"]["verify_data"]["content_matches"] is True + + +def test_snapshot_lifecycle_content_mismatch_fails( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A corrupted restore (sentinel mismatch) fails verify_data.""" + module = _load_script("snapshot_lifecycle.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2()) + _patch_ebs_volume_ops(monkeypatch, module) + monkeypatch.setattr(module.ebs, "create_snapshot", lambda *a, **k: "snap-1") + monkeypatch.setattr(module.ebs, "wait_for_snapshot_completed", lambda *a, **k: None) + monkeypatch.setattr(module.ebs, "mount_and_read_sentinel", lambda *a, **k: (0, "CORRUPT", "")) + monkeypatch.setattr(module.ebs, "detach_and_delete_volume", lambda *a, **k: None) + monkeypatch.setattr(module.ebs, "delete_snapshot_best_effort", lambda *a, **k: None) + monkeypatch.setattr(module, "wait_for_ssh", lambda *a, **k: True) + monkeypatch.setattr(module, "ssh_run", lambda *a, **k: (0, "", "")) + monkeypatch.setattr( + sys, + "argv", + [ + "snapshot_lifecycle.py", + "--instance-id", + "i-fixture", + "--volume-id", + "vol-fixture", + "--key-file", + "/tmp/k.pem", + "--expected-content", + EXPECTED_CONTENT, + ], + ) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 1 + assert payload["success"] is False + assert payload["operations"]["verify_data"]["content_matches"] is False + + +def test_snapshot_lifecycle_cleanup_error_fails_step( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A leaked restore volume / snapshot makes the snapshot step fail.""" + module = _load_script("snapshot_lifecycle.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2()) + _patch_ebs_volume_ops(monkeypatch, module) + monkeypatch.setattr(module.ebs, "create_snapshot", lambda *a, **k: "snap-1") + monkeypatch.setattr(module.ebs, "wait_for_snapshot_completed", lambda *a, **k: None) + monkeypatch.setattr(module.ebs, "mount_and_read_sentinel", lambda *a, **k: (0, EXPECTED_CONTENT, "")) + monkeypatch.setattr(module.ebs, "detach_and_delete_volume", lambda *a, **k: "DeleteVolume failed") + monkeypatch.setattr(module.ebs, "delete_snapshot_best_effort", lambda *a, **k: None) + monkeypatch.setattr(module, "wait_for_ssh", lambda *a, **k: True) + monkeypatch.setattr(module, "ssh_run", lambda *a, **k: (0, "", "")) + monkeypatch.setattr( + sys, + "argv", + [ + "snapshot_lifecycle.py", + "--instance-id", + "i-fixture", + "--volume-id", + "vol-fixture", + "--key-file", + "/tmp/k.pem", + "--expected-content", + EXPECTED_CONTENT, + ], + ) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 1 + assert payload["success"] is False + assert payload["cleanup_errors"] == ["DeleteVolume failed"] + + +# --------------------------------------------------------------------------- +# volume_resize.py +# --------------------------------------------------------------------------- + + +def test_volume_resize_happy_path(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + """ModifyVolume + growpart + resize2fs + a larger filesystem all pass.""" + module = _load_script("volume_resize.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2(volume_size=10)) + monkeypatch.setattr(module.ebs, "wait_for_modification_complete", lambda *a, **k: "completed") + monkeypatch.setattr(module, "wait_for_ssh", lambda *a, **k: True) + + sizes = iter(["10000000000", "15000000000"]) + + def fake_ssh(host: str, user: str, key: str, script: str, **kwargs: Any) -> tuple[int, str, str]: + if "df -B1" in script: + return (0, next(sizes), "") + return (0, "", "") + + monkeypatch.setattr(module, "ssh_run", fake_ssh) + monkeypatch.setattr( + sys, + "argv", + ["volume_resize.py", "--instance-id", "i-fixture", "--volume-id", "vol-fixture", "--key-file", "/tmp/k.pem"], + ) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert payload["success"] is True + assert payload["new_size_gib"] == 15 + assert all(op["passed"] for op in payload["operations"].values()) + + +def test_volume_resize_growpart_failure_fails_step( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A failing growpart fails the resize and skips resize2fs.""" + module = _load_script("volume_resize.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2(volume_size=10)) + monkeypatch.setattr(module.ebs, "wait_for_modification_complete", lambda *a, **k: "completed") + monkeypatch.setattr(module, "wait_for_ssh", lambda *a, **k: True) + + def fake_ssh(host: str, user: str, key: str, script: str, **kwargs: Any) -> tuple[int, str, str]: + if "df -B1" in script: + return (0, "10000000000", "") + if "growpart" in script: + return (1, "", "growpart boom") + return (0, "", "") + + monkeypatch.setattr(module, "ssh_run", fake_ssh) + monkeypatch.setattr( + sys, + "argv", + ["volume_resize.py", "--instance-id", "i-fixture", "--volume-id", "vol-fixture", "--key-file", "/tmp/k.pem"], + ) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 1 + assert payload["success"] is False + assert payload["operations"]["grow_partition"]["passed"] is False + assert payload["operations"]["resize_filesystem"]["passed"] is False + + +# --------------------------------------------------------------------------- +# volume_persistence.py +# --------------------------------------------------------------------------- + + +def test_volume_persistence_happy_path(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + """Stop/start with the volume still attached and data intact passes.""" + module = _load_script("volume_persistence.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2(attached_instance="i-fixture")) + monkeypatch.setattr(module.ebs, "is_volume_attached_to", lambda *a, **k: True) + monkeypatch.setattr(module.ebs, "mount_and_read_sentinel", lambda *a, **k: (0, EXPECTED_CONTENT, "")) + monkeypatch.setattr(module, "wait_for_ssh", lambda *a, **k: True) + monkeypatch.setattr(module, "wait_for_public_ip", lambda *a, **k: "203.0.113.10") + monkeypatch.setattr( + sys, + "argv", + [ + "volume_persistence.py", + "--instance-id", + "i-fixture", + "--volume-id", + "vol-fixture", + "--key-file", + "/tmp/k.pem", + "--expected-content", + EXPECTED_CONTENT, + ], + ) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert payload["success"] is True + assert all(op["passed"] for op in payload["operations"].values()) + + +def test_volume_persistence_detached_after_restart_fails( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A volume that does not reattach after restart fails verify_attached.""" + module = _load_script("volume_persistence.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2()) + monkeypatch.setattr(module.ebs, "is_volume_attached_to", lambda *a, **k: False) + monkeypatch.setattr(module.ebs, "mount_and_read_sentinel", lambda *a, **k: (0, EXPECTED_CONTENT, "")) + monkeypatch.setattr(module, "wait_for_ssh", lambda *a, **k: True) + monkeypatch.setattr(module, "wait_for_public_ip", lambda *a, **k: "203.0.113.10") + monkeypatch.setattr( + sys, + "argv", + [ + "volume_persistence.py", + "--instance-id", + "i-fixture", + "--volume-id", + "vol-fixture", + "--key-file", + "/tmp/k.pem", + "--expected-content", + EXPECTED_CONTENT, + ], + ) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 1 + assert payload["success"] is False + assert payload["operations"]["verify_attached"]["passed"] is False + + +# --------------------------------------------------------------------------- +# teardown_volume.py +# --------------------------------------------------------------------------- + + +def test_teardown_volume_deletes(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + """A successful detach + delete reports the volume as deleted.""" + module = _load_script("teardown_volume.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2()) + monkeypatch.setattr(module.ebs, "detach_and_delete_volume", lambda *a, **k: None) + monkeypatch.setattr(sys, "argv", ["teardown_volume.py", "--volume-id", "vol-fixture"]) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert payload["success"] is True + assert payload["resources_deleted"] == ["vol-fixture"] + + +def test_teardown_volume_skip_destroy(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + """--skip-destroy preserves the volume and short-circuits cleanup.""" + module = _load_script("teardown_volume.py") + monkeypatch.setattr(sys, "argv", ["teardown_volume.py", "--volume-id", "vol-fixture", "--skip-destroy"]) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert payload["success"] is True + assert payload["resources_deleted"] == []