feat(nico): add AUTH-XX-03 specified-key access validation

isvctl/configs/providers/nico/config/bare_metal.yaml

@@ -52,13 +52,22 @@
 #     nico-admin-cli into the provider-neutral attestation contract (SEC22-01 /
 #     CNP09-02). It requires operator/admin CLI credentials in addition to the
 #     tenant REST credentials used for machine enumeration.
+#   - query_key_access.py reads the site's SSH Key Groups and serial-console
+#     configuration from the NICo API and reports whether a
+#     tenant-specified key can access out-of-band components (serial console /
+#     SOL is verifiable; network devices are provider-managed). The
+#     setup_key_access.py and teardown_key_access.py steps provision a
+#     throwaway key group for the run, then remove it, so
+#     SpecifiedKeyAccessCheck (AUTH-XX-03) can validate the path without
+#     persistent manual key setup.
 #
 # Prerequisites:
 #   - NICO_API_BASE set to the NICo API base URL
 #   - NICO_BEARER_TOKEN set, or OIDC client credentials configured via
 #     NICO_SSA_ISSUER, NICO_CLIENT_ID, and NICO_CLIENT_SECRET
 #   - Expected machines pre-registered; machines ingested and DPUs initialized
 #   - NICO_ORGANIZATION and NICO_SITE_ID environment variables set
+#   - ssh-keygen available on the host (used to mint the throwaway key)
 #
 # Usage:
 #   NICO_BEARER_TOKEN=<token> NICO_ORGANIZATION=<org-name> NICO_SITE_ID=<uuid> \
@@ -71,8 +80,26 @@ version: "1.0"
 
 commands:
   bare_metal:
-    phases: ["test"]
+    phases: ["setup", "test", "teardown"]
     steps:
+      # ─── Provision a Throwaway Specified Key (AUTH-XX-03 setup) ──────
+      # Creates an ephemeral SSH key and a key group that syncs it to the
+      # site. Emits sshkey_id / sshkeygroup_id / restore_ssh_keys_enabled for
+      # teardown to consume.
+      - name: setup_key_access
+        phase: setup
+        command: "python ../scripts/auth/setup_key_access.py"
+        requires_available_validations:
+          - SpecifiedKeyAccessCheck
+        args:
+          - "--org"
+          - "{{org}}"
+          - "--site-id"
+          - "{{site_id}}"
+          - "--api-base"
+          - "{{nico_api_base}}"
+        timeout: 240
+
       # ─── Verify Hardware Ingestion ─────────────────────────────────
       # Compares expected-machine manifest against discovered machines.
       # Output includes per-machine status, health, GPU/DPU counts.
@@ -223,6 +250,50 @@ commands:
           - "{{nico_api_base}}"
         timeout: 120
 
+      # ─── Specified-Key Access (AUTH-XX-03) ──────────────────────────
+      # Reads the site's SSH Key Groups and serial-console configuration from
+      # the NICo API (/carbide/ segment, like the other NICo steps) and reports
+      # whether a tenant-specified key can access out-of-band components. The serial
+      # console (SOL) is verifiable from the tenant API (provider-enabled +
+      # SSH-key auth + key synced); network-device access is provider-managed
+      # and reported as unverified. SpecifiedKeyAccessCheck validates it.
+      - name: query_key_access
+        phase: test
+        command: "python ../scripts/auth/query_key_access.py"
+        requires_available_validations:
+          - SpecifiedKeyAccessCheck
+        args:
+          - "--org"
+          - "{{org}}"
+          - "--site-id"
+          - "{{site_id}}"
+          - "--api-base"
+          - "{{nico_api_base}}"
+        timeout: 120
+
+      # ─── Remove the Throwaway Specified Key (AUTH-XX-03 teardown) ────
+      # Best-effort cleanup of the key group + key created in setup, and
+      # restores the site's SSH-key SOL flag when setup flipped it.
+      - name: teardown_key_access
+        phase: teardown
+        command: "python ../scripts/auth/teardown_key_access.py"
+        requires_available_validations:
+          - SpecifiedKeyAccessCheck
+        args:
+          - "--org"
+          - "{{org}}"
+          - "--site-id"
+          - "{{site_id}}"
+          - "--api-base"
+          - "{{nico_api_base}}"
+          - "--sshkeygroup-id"
+          - "{{steps.setup_key_access.sshkeygroup_id}}"
+          - "--sshkey-id"
+          - "{{steps.setup_key_access.sshkey_id}}"
+          - "--restore-ssh-keys-enabled"
+          - "{{steps.setup_key_access.restore_ssh_keys_enabled}}"
+        timeout: 120
+
 tests:
   cluster_name: "nico-bare-metal-validation"
   description: "Verify NICo bare-metal hardware ingestion and DPU health"

isvctl/configs/providers/nico/scripts/auth/query_key_access.py

@@ -0,0 +1,282 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Report specified-key access to out-of-band components for a NICo site (AUTH-XX-03).
+
+AUTH-XX-03 verifies that a tenant-specified key (an SSH key) can be used to
+access other components "as possible", with the serial console (SOL) and
+network devices called out as examples.
+
+NICo models a tenant-specified key as an SSH Key, grouped into an SSH Key Group
+that is synced down to one or more Sites. A Site exposes a serial console (SOL)
+whose access the provider enables (``isSerialConsoleEnabled``) and whose
+SSH-key authentication the tenant enables (``isSerialConsoleSSHKeysEnabled``).
+When a key group with at least one key is synced to a site whose serial console
+is enabled with SSH-key auth, that key grants SOL access to the site's machines.
+
+This script gathers the provider-neutral evidence for that access path:
+
+- **specified keys** -- the SSH keys in the key groups synced to the site. Only
+  the count and fingerprint-derived posture are surfaced; key material is never
+  emitted.
+- **serial console (SOL)** -- per-site serial-console configuration and whether
+  the synced key can reach it.
+- **network devices** -- reported as ``key_access_enabled: null`` (unverified):
+  tenant key access to switches is provider-managed and not exposed by the NICo
+  tenant REST API, so it is neither asserted nor falsely passed.
+
+NICo API endpoints used (the ``/carbide/`` segment is the current deployed name
+for what newer docs call ``/nico/``; the other NICo scripts use it too):
+  GET /{org}/carbide/site/{site_id}
+  GET /{org}/carbide/sshkeygroup?siteId={site_id}
+
+Auth:
+  - NICO_BEARER_TOKEN, or OIDC client_credentials
+    (NICO_SSA_ISSUER / NICO_CLIENT_ID / NICO_CLIENT_SECRET).
+
+When no SSH key group with a key is synced to the site there is nothing to
+evidence access with, so the script emits a structured skip (``skipped: true``
++ ``skip_reason``) carrying an ``org_key_groups`` count, distinguishing "no key
+groups exist at all" from "key groups exist but none are synced to this site".
+
+Required JSON output fields:
+  {
+    "success": true,
+    "platform": "nico",
+    "site_id": "...",
+    "specified_keys": 1,
+    "key_groups": 1,
+    "keys_synced_to_site": true,
+    "access_targets": [
+      {
+        "type": "serial_console",
+        "name": "<site> serial console (SOL)",
+        "key_access_enabled": true,   // tri-state: true | false | null (unverified)
+        "reachable": true,            // endpoint present AND key synced to site
+        "detail": "..."
+      },
+      {
+        "type": "network_device",
+        "name": "...",
+        "key_access_enabled": null,
+        "reachable": false,
+        "detail": "..."
+      }
+    ]
+  }
+
+Usage:
+    NICO_BEARER_TOKEN=<token> \
+        python query_key_access.py --org <org> --site-id <uuid> --api-base <url>
+
+    Wired via the bare_metal suite:
+      uv run isvctl test run -f isvctl/configs/providers/nico/config/bare_metal.yaml
+
+Reference:
+    infra-controller rest-api/openapi/spec.yaml (Site, SshKeyGroup schemas)
+"""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+# Allow importing from sibling common/ directory
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from common.nico_client import NicoAuthError, forge_get, forge_get_all, resolve_auth
+
+# SSH Key Group / site-association status that means the key has fully
+# propagated to the site and is therefore usable for access.
+SYNCED_STATUS = "Synced"
+
+
+def _count_specified_keys(groups: list[dict[str, Any]]) -> int:
+    """Count distinct SSH keys across the key groups synced to the site."""
+    key_ids: set[str] = set()
+    keys_without_id = 0
+    for group in groups:
+        for key in group.get("sshKeys") or []:
+            key_id = key.get("id") if isinstance(key, dict) else None
+            if isinstance(key_id, str) and key_id:
+                key_ids.add(key_id)
+            else:
+                keys_without_id += 1
+    return len(key_ids) + keys_without_id
+
+
+def _keys_synced_to_site(groups: list[dict[str, Any]], site_id: str) -> bool:
+    """Return whether a key group with at least one key is synced to the site.
+
+    A key is usable on the site once its group's association reaches ``Synced``.
+    The group is accepted when either its own status is ``Synced`` or the
+    association for this specific site is ``Synced``.
+    """
+    for group in groups:
+        if not (group.get("sshKeys") or []):
+            continue
+        if group.get("status") == SYNCED_STATUS:
+            return True
+        for assoc in group.get("siteAssociations") or []:
+            if not isinstance(assoc, dict):
+                continue
+            assoc_site = (assoc.get("site") or {}).get("id")
+            if assoc_site == site_id and assoc.get("status") == SYNCED_STATUS:
+                return True
+    return False
+
+
+def _serial_console_target(site: dict[str, Any], keys_synced: bool) -> dict[str, Any]:
+    """Build the serial-console (SOL) access target from the site config."""
+    name = f"{site.get('name') or site.get('id') or 'site'} serial console (SOL)"
+
+    if not site.get("isSerialConsoleEnabled"):
+        return {
+            "type": "serial_console",
+            "name": name,
+            "key_access_enabled": None,
+            "reachable": False,
+            "detail": "Provider has not enabled the serial console (SOL) for this site",
+        }
+
+    key_access_enabled = bool(site.get("isSerialConsoleSSHKeysEnabled"))
+    hostname = (site.get("serialConsoleHostname") or "").strip()
+    reachable = bool(hostname) and keys_synced
+
+    if not key_access_enabled:
+        detail = "Serial console is enabled but SSH-key access is disabled (isSerialConsoleSSHKeysEnabled is false)"
+    elif not hostname:
+        detail = "Serial console SSH-key access is enabled but no serial console hostname is configured"
+    elif not keys_synced:
+        detail = "Serial console SSH-key access is enabled but no specified key is synced to this site"
+    else:
+        detail = "Specified key can access the serial console (SOL): provider-enabled, SSH-key auth on, key synced"
+
+    return {
+        "type": "serial_console",
+        "name": name,
+        "key_access_enabled": key_access_enabled,
+        "reachable": reachable,
+        "detail": detail,
+    }
+
+
+def _network_device_target() -> dict[str, Any]:
+    """Build the (unverified) network-device access target.
+
+    Tenant key access to network switches is provider-managed and is not
+    exposed by the NICo tenant REST API, so it is reported as unverified rather
+    than asserted or falsely passed.
+    """
+    return {
+        "type": "network_device",
+        "name": "Network devices",
+        "key_access_enabled": None,
+        "reachable": False,
+        "detail": (
+            "Tenant key access to network devices is provider-managed and not exposed by the "
+            "NICo tenant REST API; not verifiable from the tenant API"
+        ),
+    }
+
+
+def main() -> int:
+    """Gather specified-key access evidence and print the JSON contract to stdout."""
+    parser = argparse.ArgumentParser(
+        description="Report specified-key (SSH) access to out-of-band components on a NICo site"
+    )
+    parser.add_argument("--org", required=True, help="NGC org name")
+    parser.add_argument("--site-id", required=True, help="NICo site UUID")
+    parser.add_argument("--api-base", required=True, help="NICo API base URL")
+    args = parser.parse_args()
+
+    result: dict[str, Any] = {
+        "success": False,
+        "platform": "nico",
+        "site_id": args.site_id,
+        "specified_keys": 0,
+        "key_groups": 0,
+        "keys_synced_to_site": False,
+        "access_targets": [],
+    }
+
+    try:
+        auth = resolve_auth()
+
+        site = forge_get(args.org, f"site/{args.site_id}", auth.token, base_url=args.api_base)
+
+        groups = forge_get_all(
+            args.org,
+            "sshkeygroup",
+            auth.token,
+            base_url=args.api_base,
+            params={"siteId": args.site_id},
+            result_key="sshKeyGroups",
+        )
+
+        keys_synced = _keys_synced_to_site(groups, args.site_id)
+
+        result["specified_keys"] = _count_specified_keys(groups)
+        result["key_groups"] = len(groups)
+        result["keys_synced_to_site"] = keys_synced
+
+        if not keys_synced:
+            # Nothing to evidence access with on this site. Query the org-wide
+            # key groups (no siteId filter) so the skip reason can distinguish
+            # "no key groups exist at all" from "key groups exist but none are
+            # synced to this site" -- the operator needs to know which.
+            org_groups = forge_get_all(
+                args.org,
+                "sshkeygroup",
+                auth.token,
+                base_url=args.api_base,
+                result_key="sshKeyGroups",
+            )
+            result["org_key_groups"] = len(org_groups)
+            result["skipped"] = True
+            if org_groups:
+                result["skip_reason"] = (
+                    f"{len(org_groups)} SSH key group(s) exist for the org but none are synced to this site; "
+                    "sync a key group containing an SSH key to the site to enable key-based SOL access"
+                )
+            else:
+                result["skip_reason"] = (
+                    "No SSH key groups exist for the org; create one with an SSH key and sync it to the site "
+                    "to enable key-based SOL access"
+                )
+            result["success"] = True
+            print(json.dumps(result, indent=2))
+            return 0
+
+        result["access_targets"] = [
+            _serial_console_target(site, keys_synced),
+            _network_device_target(),
+        ]
+        result["success"] = True
+
+    except NicoAuthError as e:
+        result["error_type"] = "auth"
+        result["error"] = str(e)
+    except Exception as e:
+        result["error"] = f"{type(e).__name__}: {e}"
+
+    print(json.dumps(result, indent=2))
+    return 0 if result["success"] else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())