From 430c9ea95163f4dd0b04a4abf46a53a3a97902b1 Mon Sep 17 00:00:00 2001 From: Brad Edwards Date: Thu, 24 Sep 2026 05:03:40 +0000 Subject: [PATCH 1/6] fix(gcp): support scalable preconfigured custom-image hosts --- ...machine-image-range-host-preflight-1896.md | 16 +- ...igured-custom-image-host-preflight-2382.md | 76 +++++++ .../engine/provisioner/config/_gce_profile.py | 7 +- .../provisioner/gcp_range_cell_outputs.py | 8 +- .../provisioner/gcp_range_cell_resources.py | 16 +- shifter/engine/provisioner/gcp_range_cells.py | 51 +++-- .../provisioner/raes_gcp_activation_apply.py | 3 + shifter/engine/provisioner/raes_gcp_apply.py | 116 +++++++++- .../provisioner/raes_gcp_apply_types.py | 3 + .../engine/provisioner/raes_gcp_destroy.py | 2 + .../raes_preconfigured_host_readiness.py | 37 +++ .../engine/provisioner/tests/test_config.py | 52 +++++ .../tests/test_gcp_range_cell_resources.py | 19 ++ .../provisioner/tests/test_gcp_range_cells.py | 42 ++++ .../provisioner/tests/test_raes_gce_image.py | 34 +++ .../provisioner/tests/test_raes_gcp_apply.py | 210 +++++++++++++++++- .../test_raes_preconfigured_host_readiness.py | 65 ++++++ .../engine/services/_raes_image.py | 24 +- .../administer/AdapterPackBindings.test.tsx | 17 ++ .../administer/AdapterPackBindings.tsx | 14 +- .../RaesImageRegistryPage.test.tsx | 24 ++ .../RaesImageRegistryPage.tsx | 19 +- .../shared/raes/operation_input_candidates.py | 10 +- .../shared/runtime_plugin_binding.py | 14 +- .../tests/engine/services/test_raes_image.py | 38 ++++ .../tests/engine/test_operation_input_raes.py | 18 ++ .../shared/raes/test_raes_operation_input.py | 18 ++ 27 files changed, 885 insertions(+), 68 deletions(-) create mode 100644 docs/architecture/gcp-preconfigured-custom-image-host-preflight-2382.md create mode 100644 shifter/engine/provisioner/raes_preconfigured_host_readiness.py create mode 100644 shifter/engine/provisioner/tests/test_raes_preconfigured_host_readiness.py diff --git a/docs/architecture/gce-machine-image-range-host-preflight-1896.md b/docs/architecture/gce-machine-image-range-host-preflight-1896.md index 6f9dc81b5..2743f7578 100644 --- a/docs/architecture/gce-machine-image-range-host-preflight-1896.md +++ b/docs/architecture/gce-machine-image-range-host-preflight-1896.md @@ -13,6 +13,13 @@ participant container/account needed by the established setup and RDP broker paths. It does not add a scenario field, scenario-id branch, package executor, or new participant access channel. +The original machine-image-only source decision is extended by #2382 for +single-boot-disk hosts: the same `preconfigured-machine-host` capability may +use an exact `projects//global/images/` custom image. Image +kind describes the Compute source; bootstrap capability describes guest +readiness. A custom image is appropriate only when the complete preconfigured +runtime is on its boot disk. Multi-disk captures retain the machine-image path. + Machine-image profiles are administrator-managed runtime configuration. Their concrete image, container, account, and service-account values do not belong in catalog content. Legacy ranges may still use the bounded deployment-owned map @@ -22,10 +29,10 @@ the latter is pinned with the adapter, pack digest, and range operation. ## Required Controls -- Accept exactly one source per profile. A normal image profile uses - `source_image`; a preconfigured host uses the exact +- Accept exactly one source per profile. A preconfigured host uses either an + exact project-qualified custom image in `source_image` or the exact `projects//global/machineImages/` form. Families and inferred - names are not accepted for machine images. + names are not accepted for either preconfigured-host source. - Replace inherited metadata, SSH material, network interfaces, external-IP posture, labels, tags, machine type, and service account at clone time. Captured disks are the only inherited resources. @@ -42,6 +49,9 @@ the latter is pinned with the adapter, pack digest, and range operation. - After create and on reconcile, set `autoDelete=true` on every attached disk. Destroy performs the same convergence before deleting the instance, so machine-image data disks cannot be orphaned. +- A custom-image host creates one explicitly auto-deleting boot disk. Preserve + Shielded VM settings, nested virtualization, fresh metadata and SSH keys, + private networking, and an explicit runtime identity (or explicit absence). - Treat the image as owning its internal realization. The fixed volatile marker, running configured participant container, and host RDP listener are boot- liveness prerequisites only. Issue #1910 additionally requires the diff --git a/docs/architecture/gcp-preconfigured-custom-image-host-preflight-2382.md b/docs/architecture/gcp-preconfigured-custom-image-host-preflight-2382.md new file mode 100644 index 000000000..b89ea5dd4 --- /dev/null +++ b/docs/architecture/gcp-preconfigured-custom-image-host-preflight-2382.md @@ -0,0 +1,76 @@ +# GCE Preconfigured Custom-Image Host Preflight (#2382) + +Status: implementation guidance + +## Decision and boundary + +`preconfigured-machine-host` describes guest realization and readiness; `image` +versus `machine-image` describes the Compute Engine source. Permit the host +capability with either an exact `projects//global/images/` custom +image or the existing exact machine-image reference. A custom image is suitable +only when the complete runtime is on its one boot disk. Keep ordinary boot-image +and machine-image behavior, and the existing generic RAES image/profile binding. +Do not introduce a third image kind or a scenario-specific selector. +This updates the machine-image-only source assumption in the #1896 preflight; +its host security and readiness controls still apply. + +The machine-image source limit is six creations per source in 60 minutes; Google +documents 20 instances per second for custom-image API/CLI creation. The latter +still has image access, quota, and capacity constraints; it is not an unbounded +fleet guarantee. See [machine-image restrictions](https://cloud.google.com/compute/docs/machine-images/create-instance-from-machine-image) +and [custom-image creation](https://cloud.google.com/compute/docs/instances/create-vm-from-custom-image). + +## Cross-cutting gates and incumbents + +| Layer | Existing boundary and required guardrail | +| --- | --- | +| Authorization and administrator binding | CMS image-registry writes use `CMS_WRITE_PERMISSIONS`, `RaesImageMappingRegisterSerializer`, and `engine.services._raes_image.upsert_raes_image_mapping`; adapter target profiles use `cms.api.runtime_plugin_packs` and `shared.runtime_plugin_binding.RuntimeTargetImageProfile`. Keep image choice in those administrator-owned bindings, frozen with the operation or plugin pin. Pack authors cannot install executable adapters or choose a host by private identity. | +| Shapes and validation | Reuse registry service validation, `shared.raes.operation_input_candidates`' closed projection, `shared.raes.image_policy.ResolvedImage`, `raes_gce_image`, `config._gce_image_keys`' closed JSON shape, and `config._gce_profile`'s profile validation. Validate source syntax by image kind, then validate complete host identity/readiness by bootstrap capability. Exact custom-image refs must exclude families, bare names, and URLs. Preserve the ordinary-image and AWS restrictions. The UI forms and API choices must express both independent dimensions. | +| Plan and persistence | Reuse `GCERangeImageProfile`, `gce_image_profile_fingerprint`, deterministic `gcp_range_cell_plan`/`raes_gcp_plan` names and labels, registry rows, operation-input candidates, and pinned plugin bindings. No second profile schema, table, range status, or scenario field. An exact resource name is not a physical image ID: retain `source_image_id` checks for prepared artifacts through `raes_substrate_observation`, and do not claim name-only pins detect delete/recreate of an image. | +| Compute request and security | Reuse `gcp_range_cell_resources.instance_resource` and `gcp_range_cells._insert_instance`. The custom-image host uses one boot disk with `source_image` and `auto_delete=true`, plus explicit `advanced_machine_features.enable_nested_virtualization=true`. Preserve current Shielded VM flags, fresh metadata/host key, private subnetwork and IP, no external IP, labels/tags, `can_ip_forward=false`, and the explicit selected service-account list or explicit empty list. Never inherit machine-image identity or metadata. Keep the existing host identity pool where the legacy path requires it; do not infer a new role or broaden IAM. | +| Guest and access | Reuse `instance_orchestrator`, `plans.preconfigured_machine_host`, `raes_guest_plan.assert_management_login_separate`, `gcp_range_cell_outputs`, guest secret ops, private SSH host-key pinning, and declared participant access. A custom-image host follows the same bounded liveness, credential installation, and `participant-readiness/v1` canary checks; it skips ordinary guest bootstrap. The fixed canary receives only validated container/user/contract/digest arguments, never a profile-supplied command or secret in argv. A bootable image alone does not establish readiness. | +| Reconcile and cleanup | Both `gcp_range_cells._ensure_instance` and `raes_gcp_apply._ensure_raes_instance` must reject an existing deterministic VM with conflicting range ownership or image/profile labels before returning its host key. The present keyed legacy check skips empty RAES image keys, so it cannot be the sole guard. Reuse shared binding checks and `gcp_range_cell_destroy`/`raes_gcp_destroy`; keep the custom-image boot disk `auto_delete=true` on create, adoption, and destroy. Machine-image disk convergence remains in place. A late ownership conflict must preserve the foreign VM and release only resources journaled as created by this apply attempt. Do not silently adopt a VM after an ambiguous insert result. | +| Errors and observability | Preserve `RaesImageMappingError`, `RaesOperationInputError`, provisioner plan errors, `shared.api.errors.api_error_response`, `log_redact.safe_log_fingerprint`, and `terraform_ops._safe_failure_message`. Fail at the owning validator before cloud mutation where possible. Log bounded identifiers/operation context; never log request metadata, SSH material, credentials, or raw provider payloads. The failure-message boundary truncates but does not redact, so new exceptions must contain safe text. | + +The only runtime configuration transport implicated by a legacy keyed profile is +`GCP_RANGE_IMAGE_KEY_PROFILES_JSON`: `shifter/installation/runtime_inventory_gcp.py`, +`scripts/gcp/render_runtime_env.py`, `config._gce_image_keys`, and the provisioner +Job admission allowlists in `platform/k8s/gcp/base` and the Helm template already +carry that variable. Keep the change inside its existing closed shape. A new +environment variable would have to pass each of those gates and the platform +environment manifest; it is unnecessary here. The provider call uses the SDK in +process, so no image reference, token, or credential belongs in process argv. + +## Gotchas and verification boundary + +- `image_kind == "image"` currently means ordinary bootstrap in the registry + service, operation-input parser, plugin profile validator, and both UI forms. + Update those gates together. Do not key host readiness or nested virtualization + solely on `source_machine_image`. +- The RAES path has an empty `image_key`; its existing legacy drift check returns + early. Verify range ownership and the full profile fingerprint on reconcile + before adopting a preconfigured image-backed host. +- The current RAES GCE apply and existing-cell activation paths do not run the + fixed participant canary before terminal readiness. Composition verification + is a different proof. Route both preconfigured source forms through the same + bounded canary, or reject that capability on any path lacking it. A host image + must also boot under the retained Shielded VM settings on a machine type that + supports nested virtualization; validation cannot assume the bake has this. +- Keep `image_kind` as the source discriminator and `bootstrap_capability` as the + guest contract. The next boot-disk-backed capability should pass through the + same profile/request seam without another image kind or copied workflow. +- Synthetic evidence must cover registry and plugin validation, closed operation + input, exact reference rejection, request shape and explicit nested feature, + RAES realization/reconcile and conflicting-VM rejection, boot-disk ownership, + readiness routing, and ordinary-image/machine-image regressions. + +## Non-goals and anti-patterns + +No image bake or promotion pipeline, extra disk support for custom-image hosts, +global retry policy for machine-image throttling, new participant channel, new +public network exposure, new service-account privilege, pack-specific code, or +new persistence or general exception hierarchy. A typed ownership conflict may +protect foreign VMs during partial cleanup. No fallback from a failed exact image +to a family, another image, or a machine image. Existing ADR boundaries on +administrator binding, range lifecycle, and participant access remain intact; +this note clarifies their intersection without changing an ADR. diff --git a/shifter/engine/provisioner/config/_gce_profile.py b/shifter/engine/provisioner/config/_gce_profile.py index 4e2facf1c..02e8c8d7c 100644 --- a/shifter/engine/provisioner/config/_gce_profile.py +++ b/shifter/engine/provisioner/config/_gce_profile.py @@ -125,6 +125,7 @@ def gce_image_profile_fingerprint(profile: GCERangeImageProfile) -> str: _GCE_MACHINE_IMAGE_REFERENCE_RE = re.compile( rf"^(?:(?:https://[^/]+/compute/(?:v1|beta)/)?)projects/{_GCE_PROJECT}/global/machineImages/{_GCE_NAME}$" ) +_GCE_EXACT_IMAGE_REFERENCE_RE = re.compile(rf"^projects/{_GCE_PROJECT}/global/images/{_GCE_NAME}$") _GCE_LINUX_USERNAME_RE = re.compile(r"[a-z_][a-z0-9_-]{0,31}") _GCE_CONTAINER_NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}") @@ -192,8 +193,10 @@ def _validate_preconfigured_machine_profile(prefix: str, profile: GCERangeImageP prefix, profile, (profile.participant_container_name, profile.participant_username, *readiness_fields) ) return - if not profile.source_machine_image: - raise RuntimeError(f"{prefix} preconfigured-machine-host requires source_machine_image") + if not _profile_has_source(profile): + raise RuntimeError(f"{prefix} preconfigured-machine-host requires a source image") + if profile.source_image and not _GCE_EXACT_IMAGE_REFERENCE_RE.fullmatch(profile.source_image): + raise RuntimeError(f"{prefix} preconfigured-machine-host requires an exact custom-image reference") if not all(identity_fields): raise RuntimeError( f"{prefix} preconfigured-machine-host requires participant_container_name, " diff --git a/shifter/engine/provisioner/gcp_range_cell_outputs.py b/shifter/engine/provisioner/gcp_range_cell_outputs.py index 993e29a6c..f219021f8 100644 --- a/shifter/engine/provisioner/gcp_range_cell_outputs.py +++ b/shifter/engine/provisioner/gcp_range_cell_outputs.py @@ -21,8 +21,8 @@ class InstanceCredentials: host_public_key: str = "" -def _machine_image_output(instance: InstancePlan) -> ResourceDict: - """Render the machine-image-only fields of a preconfigured range host.""" +def _preconfigured_host_output(instance: InstancePlan) -> ResourceDict: + """Render participant-host fields independently of the GCE source kind.""" return { "gcp_source_machine_image": instance["profile"].source_machine_image, "gcp_participant_container_name": instance["profile"].participant_container_name, @@ -103,8 +103,8 @@ def instance_output( "gcp_bootstrap_capability": instance["profile"].bootstrap_capability, "gcp_service_account_email": _service_account_output(instance, config), } - if instance["profile"].source_machine_image: - output.update(_machine_image_output(instance)) + if instance["profile"].bootstrap_capability == GCE_BOOTSTRAP_PRECONFIGURED_MACHINE_HOST: + output.update(_preconfigured_host_output(instance)) # The image's declared Guacamole SFTP root travels as realized per-instance # metadata (#375) so Mission Control consumes it instead of an OS map. Emitted # only when the profile declares one; a blank profile emits no key so the diff --git a/shifter/engine/provisioner/gcp_range_cell_resources.py b/shifter/engine/provisioner/gcp_range_cell_resources.py index 26f686a00..0bc16b4cc 100644 --- a/shifter/engine/provisioner/gcp_range_cell_resources.py +++ b/shifter/engine/provisioner/gcp_range_cell_resources.py @@ -227,12 +227,11 @@ def instance_resource( "deletion_protection": False, "can_ip_forward": False, } - if profile.source_machine_image: - # The machine image supplies every captured disk. Network, metadata, - # identity, labels, tags, machine type, and external-IP posture are all - # explicitly replaced by the body above. + if profile.bootstrap_capability == "preconfigured-machine-host": body["advanced_machine_features"] = {"enable_nested_virtualization": True} - else: + # A machine image supplies its captured disks; a custom image needs an + # explicitly owned boot disk with the same host hardening. + if not profile.source_machine_image: body["disks"] = [ { "boot": True, @@ -259,9 +258,8 @@ def instance_resource( "scopes": list(config.service_account_scopes), } ] - elif profile.source_machine_image: - # An omitted field inherits the captured machine-image identity. Send an - # explicit empty list when this range node has no authorized runtime - # identity so the bake-time service account is never attached. + elif profile.source_machine_image or profile.bootstrap_capability == "preconfigured-machine-host": + # Do not inherit a bake-time machine-image identity. An explicit empty + # list also records the administrator's choice for a custom-image host. body["service_accounts"] = [] return body diff --git a/shifter/engine/provisioner/gcp_range_cells.py b/shifter/engine/provisioner/gcp_range_cells.py index 8db330552..a4c9f2719 100644 --- a/shifter/engine/provisioner/gcp_range_cells.py +++ b/shifter/engine/provisioner/gcp_range_cells.py @@ -6,7 +6,7 @@ import logging from collections.abc import Callable -from config import GCERangeCellConfig, load_gce_range_cell_config +from config import GCE_BOOTSTRAP_PRECONFIGURED_MACHINE_HOST, GCERangeCellConfig, load_gce_range_cell_config from gcp_range_cell_clients import GCEClients, _build_clients from gcp_range_cell_credentials import ( GCEGuestSecretOps, @@ -15,6 +15,7 @@ _default_vertex_ops, ) from gcp_range_cell_destroy import destroy_range_cell +from gcp_range_cell_naming import _label_value from gcp_range_cell_ops import _get_or_none, _wait_for_operation from gcp_range_cell_outputs import InstanceCredentials, instance_output, range_cell_result, subnet_outputs from gcp_range_cell_plan import render_range_cell_plan @@ -51,18 +52,23 @@ logger = logging.getLogger(__name__) -def _ensure_network(plan: RangeCellPlan, clients: GCEClients) -> None: +class GCEInstanceBindingError(RuntimeError): + """An existing deterministic VM does not belong to this range profile.""" + + +def _ensure_network(plan: RangeCellPlan, clients: GCEClients) -> bool: """Create the range VPC if it is missing.""" name = plan["network"]["name"] existing = _get_or_none(clients.networks.get, clients.google_exceptions, project=plan["project_id"], network=name) if existing is not None: logger.info("GCE range network exists name_fp=%s", safe_log_fingerprint(name)) - return + return False operation = clients.networks.insert(project=plan["project_id"], network_resource=network_resource(plan)) _wait_for_operation(plan, clients, operation, "global") + return True -def _ensure_subnetwork(plan: RangeCellPlan, clients: GCEClients, subnet: SubnetPlan) -> None: +def _ensure_subnetwork(plan: RangeCellPlan, clients: GCEClients, subnet: SubnetPlan) -> bool: """Create a range subnetwork if it is missing.""" name = subnet["resource_name"] existing = _get_or_none( @@ -74,16 +80,17 @@ def _ensure_subnetwork(plan: RangeCellPlan, clients: GCEClients, subnet: SubnetP ) if existing is not None: logger.info("GCE range subnetwork exists name_fp=%s", safe_log_fingerprint(name)) - return + return False operation = clients.subnetworks.insert( project=plan["project_id"], region=plan["region"], subnetwork_resource=subnetwork_resource(plan, subnet), ) _wait_for_operation(plan, clients, operation, "region") + return True -def _ensure_firewall(plan: RangeCellPlan, clients: GCEClients, firewall: FirewallPlan) -> None: +def _ensure_firewall(plan: RangeCellPlan, clients: GCEClients, firewall: FirewallPlan) -> bool: """Create one range firewall rule, or reconcile an existing rule to the plan. Name existence is not correctness (#1711 / ADR-039-R9): a rule that already @@ -100,13 +107,14 @@ def _ensure_firewall(plan: RangeCellPlan, clients: GCEClients, firewall: Firewal if existing is None: operation = clients.firewalls.insert(project=plan["project_id"], firewall_resource=body) _wait_for_operation(plan, clients, operation, "global") - return + return True logger.info("GCE range firewall reconcile name_fp=%s", safe_log_fingerprint(name)) operation = clients.firewalls.patch(project=plan["project_id"], firewall=name, firewall_resource=body) _wait_for_operation(plan, clients, operation, "global") + return False -def _ensure_router_nat(plan: RangeCellPlan, clients: GCEClients) -> None: +def _ensure_router_nat(plan: RangeCellPlan, clients: GCEClients) -> bool: """Create the range-owned Cloud Router + NAT if the plan carries one (PLAT-238). Present only for a non-``none`` range; a zero-egress range has no ``router_nat`` @@ -115,7 +123,7 @@ def _ensure_router_nat(plan: RangeCellPlan, clients: GCEClients) -> None: """ router_nat = plan.get("router_nat") if router_nat is None: - return + return False name = router_nat["router_name"] existing = _get_or_none( clients.routers.get, @@ -126,16 +134,17 @@ def _ensure_router_nat(plan: RangeCellPlan, clients: GCEClients) -> None: ) if existing is not None: logger.info("GCE range router/NAT exists name_fp=%s", safe_log_fingerprint(name)) - return + return False operation = clients.routers.insert( project=plan["project_id"], region=plan["region"], router_resource=router_nat_resource(plan), ) _wait_for_operation(plan, clients, operation, "region") + return True -def _ensure_address(plan: RangeCellPlan, clients: GCEClients, instance: InstancePlan) -> None: +def _ensure_address(plan: RangeCellPlan, clients: GCEClients, instance: InstancePlan) -> bool: """Reserve an internal address for one range instance.""" name = instance["address_name"] existing = _get_or_none( @@ -147,13 +156,14 @@ def _ensure_address(plan: RangeCellPlan, clients: GCEClients, instance: Instance ) if existing is not None: logger.info("GCE range address exists name_fp=%s", safe_log_fingerprint(name)) - return + return False operation = clients.addresses.insert( project=plan["project_id"], region=plan["region"], address_resource=address_resource(instance), ) _wait_for_operation(plan, clients, operation, "region") + return True def _host_public_key_from_instance(existing: object) -> str: @@ -192,6 +202,20 @@ def _assert_instance_image_binding(existing: object, instance: InstancePlan) -> ) +def _assert_preconfigured_host_binding(existing: object, plan: RangeCellPlan, instance: InstancePlan) -> None: + """Never adopt a deterministic participant host with different ownership or image policy.""" + if instance["profile"].bootstrap_capability != GCE_BOOTSTRAP_PRECONFIGURED_MACHINE_HOST: + return + expected = { + "managed-by": plan["labels"]["managed-by"], + "range-id": plan["labels"]["range-id"], + "image-key": _label_value(instance["image_key"] or "default"), + "image-profile": instance["image_profile_fingerprint"], + } + if any(_existing_label(existing, key) != value for key, value in expected.items()): + raise GCEInstanceBindingError("Existing GCE participant host has a conflicting range or image-profile binding") + + def _ensure_attached_disks_auto_delete( plan: RangeCellPlan, clients: GCEClients, @@ -271,6 +295,7 @@ def _ensure_instance( instance=name, ) if existing is not None: + _assert_preconfigured_host_binding(existing, plan, instance) _assert_instance_image_binding(existing, instance) host_secret_ref, host_management_public_key = secret_ops.ensure_ssh(plan["range_id"], instance["source"]) access_channels = set(instance["participant_access_channels"]) @@ -291,7 +316,7 @@ def _ensure_instance( rdp_password_secret_ref, _password = secret_ops.ensure_rdp_password(plan["range_id"], instance["source"]) if existing is not None: logger.info("GCE range instance exists name_fp=%s", safe_log_fingerprint(name)) - if instance["profile"].source_machine_image: + if instance["profile"].bootstrap_capability == GCE_BOOTSTRAP_PRECONFIGURED_MACHINE_HOST: _ensure_attached_disks_auto_delete(plan, clients, name, existing) return ( host_secret_ref, diff --git a/shifter/engine/provisioner/raes_gcp_activation_apply.py b/shifter/engine/provisioner/raes_gcp_activation_apply.py index 9cadf3f83..23305fa6b 100644 --- a/shifter/engine/provisioner/raes_gcp_activation_apply.py +++ b/shifter/engine/provisioner/raes_gcp_activation_apply.py @@ -15,6 +15,7 @@ _apply_runtime, _assert_content_delivery_bindings_complete, _bootstrap_by_node, + _preflight_existing_hosts, _provision_raes_resources, _realize_directory, ) @@ -55,6 +56,7 @@ def realize_existing_cell( ) for instance in plan["instances"]: assert_management_login_separate(raes_plan, instance["uuid"].rsplit("#", 1)[0], instance["host_ssh_username"]) + _preflight_existing_hosts(plan, runtime.clients) outputs = _provision_raes_resources( plan, runtime, @@ -71,6 +73,7 @@ def realize_existing_cell( ) verified.update(item.address for item in raes_plan.content if item.source_name) verified.update(feature.address for feature in raes_plan.features) + runtime.host_readiness_verifier(outputs) observe_participant_host_keys(outputs) verified.update(runtime.composition_verifier(raes_plan, outputs)) operating_systems = runtime.operating_system_observer(raes_plan, outputs) diff --git a/shifter/engine/provisioner/raes_gcp_apply.py b/shifter/engine/provisioner/raes_gcp_apply.py index 339afa9d9..bcbd0cd5f 100644 --- a/shifter/engine/provisioner/raes_gcp_apply.py +++ b/shifter/engine/provisioner/raes_gcp_apply.py @@ -31,11 +31,13 @@ from config import GCERangeCellConfig, GCERangeImageProfile, load_gce_range_cell_config from gcp_range_cell_clients import GCEClients, _build_clients -from gcp_range_cell_ops import _get_or_none +from gcp_range_cell_ops import _delete_resource, _get_or_none from gcp_range_cell_outputs import InstanceCredentials, instance_output, subnet_outputs from gcp_range_cell_resources import instance_resource from gcp_range_cell_types import GceEgressPolicy, InstancePlan, RangeCellPlan, ResourceDict from gcp_range_cells import ( + GCEInstanceBindingError, + _assert_preconfigured_host_binding, _ensure_address, _ensure_attached_disks_auto_delete, _ensure_firewall, @@ -48,6 +50,7 @@ from raes_access import RealizedAccessBinding, join_participant_access from raes_account_credentials import ( default_account_credential_ops, + delete_instance_account_credentials, ) from raes_active_directory import ( default_directory_secret_ops, @@ -58,7 +61,7 @@ from raes_content_delivery import assert_content_delivery_bindings_complete from raes_gcp_apply_types import RaesGceApplyOptions, RaesGceApplyRuntime from raes_gcp_composition import node_bootstrap_script -from raes_gcp_destroy import RaesGceDestroyOptions, destroy_raes_range_cell +from raes_gcp_destroy import RaesGceDestroyOptions, _instance_accounts, destroy_raes_range_cell from raes_gcp_plan import RaesGcePlanOptions, build_raes_range_cell_plan from raes_gcp_secret_ops import RaesGceSecretOps, _default_secret_ops from raes_guest_plan import ( @@ -96,6 +99,7 @@ def _apply_runtime( composition_verifier=options.composition_verifier, operating_system_observer=options.operating_system_observer, substrate_observer=options.substrate_observer, + host_readiness_verifier=options.host_readiness_verifier, allocated_network_cidrs=options.allocated_network_cidrs, runtime_plugin=options.runtime_plugin, model_enrollment=options.model_enrollment, @@ -128,6 +132,7 @@ def _ensure_raes_instance( instance: InstancePlan, secret_ops: RaesGceSecretOps, bootstrap_by_node: dict[str, str], + created: list[tuple[str, str]] | None = None, ) -> tuple[str, str, str]: """Create one RAES range instance with a provisioner-managed SSH + host key. @@ -149,9 +154,11 @@ def _ensure_raes_instance( ) if existing is None: verify_prepared_source(plan, instance, clients) + else: + _assert_preconfigured_host_binding(existing, plan, instance) secret_ref, public_key = secret_ops.ensure_ssh(plan["range_id"], instance["uuid"]) if existing is not None: - if instance["profile"].source_machine_image: + if instance["profile"].bootstrap_capability == "preconfigured-machine-host": _ensure_attached_disks_auto_delete(plan, clients, name, existing) return secret_ref, public_key, _host_public_key_from_instance(existing) @@ -171,6 +178,8 @@ def _ensure_raes_instance( composition_script=bootstrap_by_node.get(_node_address_of(instance), ""), ), ) + if created is not None: + created.append(("instance", name)) return secret_ref, public_key, host_public_key @@ -180,6 +189,7 @@ def _provision_raes_resources( bootstrap_by_node: dict[str, str], accounts_by_node: dict[str, tuple[RaesPlanAccount, ...]], access_by_node: dict[str, tuple[RealizedAccessBinding, ...]], + created: list[tuple[str, str]] | None = None, ) -> list[ResourceDict]: """Create the network, subnets, firewalls, and instances for an RAES range. @@ -187,16 +197,20 @@ def _provision_raes_resources( authored account credential installed and verified on the guest, so a declared endpoint never appears with a credential that was never realized. """ - if plan["manage_network"]: - _ensure_network(plan, runtime.clients) + if plan["manage_network"] and _ensure_network(plan, runtime.clients) and created is not None: + created.append(("network", plan["network"]["name"])) for subnet in plan["subnets"]: - _ensure_subnetwork(plan, runtime.clients, subnet) - _ensure_router_nat(plan, runtime.clients) + if _ensure_subnetwork(plan, runtime.clients, subnet) and created is not None: + created.append(("subnetwork", subnet["resource_name"])) + if _ensure_router_nat(plan, runtime.clients) and created is not None: + created.append(("router", plan["router_nat"]["router_name"])) for firewall in plan["firewalls"]: - _ensure_firewall(plan, runtime.clients, firewall) + if _ensure_firewall(plan, runtime.clients, firewall) and created is not None: + created.append(("firewall", firewall["name"])) instance_outputs: list[ResourceDict] = [] for instance in plan["instances"]: - _ensure_address(plan, runtime.clients, instance) + if _ensure_address(plan, runtime.clients, instance) and created is not None: + created.append(("address", instance["address_name"])) ssh_secret_ref, ssh_public_key, host_public_key = _ensure_raes_instance( plan, runtime.clients, @@ -204,6 +218,7 @@ def _provision_raes_resources( instance, runtime.secret_ops, bootstrap_by_node, + created, ) output = instance_output( plan, @@ -237,6 +252,75 @@ def _provision_raes_resources( return instance_outputs +def _cleanup_created_resources( + plan: RangeCellPlan, + raes_plan: RaesPlan, + runtime: RaesGceApplyRuntime, + created: list[tuple[str, str]], +) -> None: + """Release only this attempt's resources after a late foreign-VM conflict.""" + clients = runtime.clients + services = { + "instance": (clients.instances, "zone", "instance", {"zone": plan["zone"]}), + "address": (clients.addresses, "region", "address", {"region": plan["region"]}), + "firewall": (clients.firewalls, "global", "firewall", {}), + "router": (clients.routers, "region", "router", {"region": plan["region"]}), + "subnetwork": (clients.subnetworks, "region", "subnetwork", {"region": plan["region"]}), + "network": (clients.networks, "global", "network", {}), + } + instances = {instance["resource_name"]: instance for instance in plan["instances"]} + for kind, name in reversed(created): + service, scope, field, extra = services[kind] + try: + if kind == "instance": + existing = _get_or_none( + clients.instances.get, + clients.google_exceptions, + project=plan["project_id"], + zone=plan["zone"], + instance=name, + ) + if existing is not None: + _ensure_attached_disks_auto_delete(plan, clients, name, existing) + _delete_resource( + plan, + clients, + service.get, + service.delete, + scope, + project=plan["project_id"], + **extra, + **{field: name}, + ) + if kind == "instance": + instance = instances[name] + runtime.secret_ops.delete_ssh(plan["range_id"], instance["uuid"]) + delete_instance_account_credentials( + plan["range_id"], + instance["uuid"], + _instance_accounts(raes_plan, instance), + runtime.account_secret_ops, + ) + except Exception as exc: + logger.error("Failed to clean attempt-created GCE resource kind=%s error_type=%s", kind, type(exc).__name__) + + +def _preflight_existing_hosts(plan: RangeCellPlan, clients: GCEClients) -> None: + """Reject conflicting deterministic hosts before any network or secret mutation.""" + for instance in plan["instances"]: + if instance["profile"].bootstrap_capability != "preconfigured-machine-host": + continue + existing = _get_or_none( + clients.instances.get, + clients.google_exceptions, + project=plan["project_id"], + zone=plan["zone"], + instance=instance["resource_name"], + ) + if existing is not None: + _assert_preconfigured_host_binding(existing, plan, instance) + + def _bootstrap_by_node(raes_plan: RaesPlan) -> dict[str, str]: """Render non-empty local composition bootstrap scripts by node.""" return {node.address: script for node in raes_plan.nodes if (script := node_bootstrap_script(node, raes_plan))} @@ -331,6 +415,8 @@ def apply_raes_range_cell( resolved_config = resolved_options.config or load_gce_range_cell_config() runtime: RaesGceApplyRuntime | None = None mutation_started = False + created: list[tuple[str, str]] = [] + plan: RangeCellPlan | None = None try: realized_access = join_participant_access(access_bindings or (), raes_plan) _assert_composition_targets_resolve(raes_plan) @@ -360,6 +446,7 @@ def apply_raes_range_cell( for instance in plan["instances"]: assert_management_login_separate(raes_plan, _node_address_of(instance), instance["host_ssh_username"]) runtime = _apply_runtime(resolved_options, config=resolved_config) + _preflight_existing_hosts(plan, runtime.clients) mutation_started = True instance_outputs = _provision_raes_resources( plan, @@ -367,6 +454,7 @@ def apply_raes_range_cell( _bootstrap_by_node(raes_plan), _accounts_by_node(raes_plan), _access_by_node(realized_access), + created, ) verified = set(_realize_directory(plan, raes_plan, instance_outputs, runtime)) verified.update(_realize_content_delivery(raes_plan, instance_outputs, delivery_bindings, runtime)) @@ -374,12 +462,22 @@ def apply_raes_range_cell( runtime.model_enrollment(raes_plan, instance_outputs) if runtime.runtime_plugin is not None: runtime.runtime_plugin(raes_plan, instance_outputs) + runtime.host_readiness_verifier(instance_outputs) observe_participant_host_keys(instance_outputs) verified.update(runtime.composition_verifier(raes_plan, instance_outputs)) operating_systems = runtime.operating_system_observer(raes_plan, instance_outputs) validate_operating_systems(raes_plan, operating_systems) compute_substrates = runtime.substrate_observer(plan, runtime.clients) snapshot_resources(raes_plan, verified) + except GCEInstanceBindingError: + # A conflicting VM is not ours to delete, even if a race placed it + # after the read-only preflight and some network resources were made. + logger.exception("RAES GCE range-cell apply found a conflicting deterministic VM request_id=%s", request_uuid) + if mutation_started and runtime is not None and plan is not None: + _cleanup_created_resources(plan, raes_plan, runtime, created) + if not mutation_started and resolved_options.on_pre_mutation_failure is not None: + resolved_options.on_pre_mutation_failure() + raise except Exception: if mutation_started and runtime is not None: logger.exception("RAES GCE range-cell apply failed; attempting cleanup request_id=%s", request_uuid) diff --git a/shifter/engine/provisioner/raes_gcp_apply_types.py b/shifter/engine/provisioner/raes_gcp_apply_types.py index 2284eafc7..0b0e704dc 100644 --- a/shifter/engine/provisioner/raes_gcp_apply_types.py +++ b/shifter/engine/provisioner/raes_gcp_apply_types.py @@ -13,6 +13,7 @@ from raes_content_delivery import realize_raes_content_delivery from raes_gcp_secret_ops import RaesGceSecretOps from raes_operating_system import observe_operating_systems +from raes_preconfigured_host_readiness import verify_preconfigured_hosts from raes_substrate_observation import observe_gce_substrates @@ -34,6 +35,7 @@ class RaesGceApplyOptions: composition_verifier: Callable[..., frozenset[str]] = verify_bootstrap_composition operating_system_observer: Callable[..., list[dict[str, str]]] = observe_operating_systems substrate_observer: Callable[..., list[dict[str, str]]] = observe_gce_substrates + host_readiness_verifier: Callable[[list[dict[str, object]]], None] = verify_preconfigured_hosts runtime_plugin: Callable[..., None] | None = None model_enrollment: Callable[..., None] | None = None @@ -55,6 +57,7 @@ class RaesGceApplyRuntime: composition_verifier: Callable[..., frozenset[str]] operating_system_observer: Callable[..., list[dict[str, str]]] substrate_observer: Callable[..., list[dict[str, str]]] + host_readiness_verifier: Callable[[list[dict[str, object]]], None] allocated_network_cidrs: Sequence[tuple[str, str]] | None runtime_plugin: Callable[..., None] | None model_enrollment: Callable[..., None] | None diff --git a/shifter/engine/provisioner/raes_gcp_destroy.py b/shifter/engine/provisioner/raes_gcp_destroy.py index f4bd11d15..21a90585f 100644 --- a/shifter/engine/provisioner/raes_gcp_destroy.py +++ b/shifter/engine/provisioner/raes_gcp_destroy.py @@ -19,6 +19,7 @@ from config import GCERangeCellConfig, GCERangeImageProfile, load_gce_range_cell_config from gcp_range_cell_clients import GCEClients, _build_clients +from gcp_range_cell_destroy import _mark_disks_auto_delete from gcp_range_cell_firewall import public_web_firewall_name from gcp_range_cell_model_broker import broker_firewall_name from gcp_range_cell_ops import _delete_resource @@ -114,6 +115,7 @@ def _destroy_instances( ) -> None: """Delete instances, addresses, and their deterministic guest secrets.""" for instance in reversed(plan["instances"]): + _mark_disks_auto_delete(plan, runtime.clients, instance["resource_name"]) _delete_resource( plan, runtime.clients, diff --git a/shifter/engine/provisioner/raes_preconfigured_host_readiness.py b/shifter/engine/provisioner/raes_preconfigured_host_readiness.py new file mode 100644 index 000000000..06fa8ac01 --- /dev/null +++ b/shifter/engine/provisioner/raes_preconfigured_host_readiness.py @@ -0,0 +1,37 @@ +"""Fixed participant-host readiness check for RAES GCE realization.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from config import GCE_BOOTSTRAP_PRECONFIGURED_MACHINE_HOST +from executors.factory import GuestExecutionContext, build_guest_execution_context +from orchestrators.setup_orchestrator import SetupError, SetupOrchestrator +from plans.preconfigured_machine_host import PreconfiguredMachineHostPlan + + +def verify_preconfigured_hosts( + instances: list[dict[str, Any]], + *, + execution_builder: Callable[..., GuestExecutionContext] = build_guest_execution_context, +) -> None: + """Gate readiness on the same bounded liveness and fixed participant canary.""" + plan = PreconfiguredMachineHostPlan() + for instance in instances: + if instance.get("gcp_bootstrap_capability") != GCE_BOOTSTRAP_PRECONFIGURED_MACHINE_HOST: + continue + execution = execution_builder( + instance, os_type=instance.get("os", "kali"), role=instance.get("role", "attacker") + ) + try: + if not execution.wait_for_ready(timeout_seconds=300): + raise SetupError("Preconfigured range host management transport is unavailable") + context = plan.get_context(instance) + result = SetupOrchestrator(executor=execution.executor).orchestrate( + execution.target, plan, context, document_name=execution.document_name + ) + if not result.success: + raise SetupError(f"Preconfigured range host failed participant readiness: {result.error}") + finally: + execution.close() diff --git a/shifter/engine/provisioner/tests/test_config.py b/shifter/engine/provisioner/tests/test_config.py index ca8d145ea..e76eba53e 100644 --- a/shifter/engine/provisioner/tests/test_config.py +++ b/shifter/engine/provisioner/tests/test_config.py @@ -797,6 +797,43 @@ def test_load_gce_range_cell_config_parses_exact_machine_image_profile(self, moc assert profile.participant_readiness_manifest_sha256 == "a" * 64 assert config.range_host_identity_pool_size == 200 + def test_load_gce_range_cell_config_parses_exact_custom_image_host(self, mocker): + mapping = { + "kali": { + "nested-host": { + "source_image": "projects/test/global/images/nested-host-v1", + "machine_type": "n2-standard-8", + "disk_size_gb": 220, + "disk_type": "pd-balanced", + "bootstrap_capability": GCE_BOOTSTRAP_PRECONFIGURED_MACHINE_HOST, + "participant_container_name": "participant-desktop", + "participant_username": "operator", + "host_ssh_username": "hostadmin", + "participant_readiness_contract": GCE_PARTICIPANT_READINESS_CONTRACT_V1, + "participant_readiness_manifest_sha256": "a" * 64, + } + } + } + mocker.patch.dict( + os.environ, + { + "CLOUD_PROVIDER": "gcp", + "GCP_RANGE_BACKEND": "gce", + "GCP_PROJECT_ID": "test-project", + "GCP_REGION": "us-central1", + "RANGE_NETWORK_ZONE": "us-central1-b", + "RANGE_NETWORK_ID": "projects/test-project/global/networks/range-net", + "GCP_RANGE_HOST_SERVICE_ACCOUNT_EMAIL": "range-host@test-project.iam.gserviceaccount.com", + "GCP_RANGE_KALI_IMAGE": "projects/test/global/images/shifter-kali", + "GCP_RANGE_IMAGE_KEY_PROFILES_JSON": json.dumps(mapping), + }, + clear=True, + ) + + profile = load_gce_range_cell_config().get_profile(role="attacker", os_type="kali", ami_key="nested-host") + assert profile.source_image == "projects/test/global/images/nested-host-v1" + assert profile.source_machine_image == "" + @pytest.mark.parametrize( ("raw", "message"), [ @@ -999,6 +1036,21 @@ def test_load_gce_range_cell_config_rejects_oversized_or_excessive_image_key_pro @pytest.mark.parametrize( ("entry", "message"), [ + ( + { + "source_image": "projects/test/global/images/family/nested-host", + "machine_type": "n2-standard-8", + "disk_size_gb": 220, + "disk_type": "pd-balanced", + "bootstrap_capability": GCE_BOOTSTRAP_PRECONFIGURED_MACHINE_HOST, + "participant_container_name": "desktop", + "participant_username": "operator", + "host_ssh_username": "hostadmin", + "participant_readiness_contract": GCE_PARTICIPANT_READINESS_CONTRACT_V1, + "participant_readiness_manifest_sha256": "a" * 64, + }, + "exact custom-image reference", + ), ( { "source_image": "projects/test/global/images/host", diff --git a/shifter/engine/provisioner/tests/test_gcp_range_cell_resources.py b/shifter/engine/provisioner/tests/test_gcp_range_cell_resources.py index 35e769b3a..c4b029e10 100644 --- a/shifter/engine/provisioner/tests/test_gcp_range_cell_resources.py +++ b/shifter/engine/provisioner/tests/test_gcp_range_cell_resources.py @@ -325,6 +325,25 @@ def test_machine_image_instance_replaces_inherited_identity_network_and_metadata assert body["service_accounts"][0]["email"] == "sh-range-host-4@test-project.iam.gserviceaccount.com" assert _metadata_map(body)["ssh-keys"] == "hostadmin:ssh-ed25519 AAAAkey" + def test_custom_image_preconfigured_host_has_nested_virt_and_owned_boot_disk(self): + instance = _instance(attach_service_account=False) + instance["profile"] = GCERangeImageProfile( + source_image="projects/test/global/images/nested-host-v1", + machine_type="n2-standard-8", + disk_size_gb=220, + bootstrap_capability="preconfigured-machine-host", + participant_container_name="participant-desktop", + participant_username="operator", + host_ssh_username="hostadmin", + ) + instance["host_ssh_username"] = "hostadmin" + body = instance_resource(_plan(), instance, _config(), ssh_public_key="ssh-ed25519 AAAAkey") + assert body["advanced_machine_features"] == {"enable_nested_virtualization": True} + assert body["disks"][0]["auto_delete"] is True + assert body["disks"][0]["initialize_params"]["source_image"] == instance["profile"].source_image + assert body["service_accounts"] == [] + assert body["shielded_instance_config"]["enable_secure_boot"] is True + def test_machine_image_instance_clears_inherited_identity_when_none_is_authorized(self): instance = _instance(attach_service_account=False) instance["profile"] = GCERangeImageProfile( diff --git a/shifter/engine/provisioner/tests/test_gcp_range_cells.py b/shifter/engine/provisioner/tests/test_gcp_range_cells.py index 0c46571b7..d6dc2bd19 100644 --- a/shifter/engine/provisioner/tests/test_gcp_range_cells.py +++ b/shifter/engine/provisioner/tests/test_gcp_range_cells.py @@ -1047,6 +1047,48 @@ def test_machine_image_instance_clone_converges_all_attached_disks_to_auto_delet ) +def test_custom_image_host_reconcile_restores_boot_disk_auto_delete(mocker): + base = _sample_config() + profile = GCERangeImageProfile( + source_image="projects/test-project/global/images/nested-host-v1", + machine_type="n2-standard-8", + disk_size_gb=220, + bootstrap_capability=GCE_BOOTSTRAP_PRECONFIGURED_MACHINE_HOST, + participant_container_name="participant-desktop", + participant_username="operator", + host_ssh_username="hostadmin", + participant_readiness_contract=GCE_PARTICIPANT_READINESS_CONTRACT_V1, + participant_readiness_manifest_sha256="a" * 64, + ) + profiles = {profile_class: dict(entries) for profile_class, entries in base.image_key_profiles.items()} + profiles["kali"]["nested-host"] = profile + config = dataclasses.replace(base, image_key_profiles=profiles, range_host_identity_pool_size=10) + payload = _scenario_payload() + payload["subnets"][0]["instances"][0]["ami_key"] = "nested-host" + plan = render_range_cell_plan("req-123", _variables(payload=payload), config, range_host_pool_slot=4) + instance = plan["instances"][0] + clients = _mock_clients(exists=False) + clients.instances.get.side_effect = None + clients.instances.get.return_value = SimpleNamespace( + labels={**plan["labels"], "image-key": "nested-host", "image-profile": instance["image_profile_fingerprint"]}, + disks=[SimpleNamespace(device_name="boot", auto_delete=False)], + metadata=SimpleNamespace(items=[]), + ) + clients.instances.set_disk_auto_delete.return_value = SimpleNamespace(name="op") + secret_ops, _ = _mock_secret_ops(mocker) + + _ensure_instance(plan, clients, config, instance, secret_ops) + + clients.instances.insert.assert_not_called() + clients.instances.set_disk_auto_delete.assert_called_once_with( + project="test-project", + zone="us-central1-b", + instance=instance["resource_name"], + device_name="boot", + auto_delete=True, + ) + + def test_render_plan_resolves_distinct_images_for_same_role_by_ami_key(): payload = _scenario_payload() example = payload["subnets"][0]["instances"][0] diff --git a/shifter/engine/provisioner/tests/test_raes_gce_image.py b/shifter/engine/provisioner/tests/test_raes_gce_image.py index 57c33785d..951629843 100644 --- a/shifter/engine/provisioner/tests/test_raes_gce_image.py +++ b/shifter/engine/provisioner/tests/test_raes_gce_image.py @@ -42,6 +42,40 @@ def _candidate(version: str, image_ref: str, **extra) -> dict: class TestRegistryResolution: + @pytest.mark.parametrize("image_ref", ["family/nested-host", "projects/example/global/images/family/nested-host"]) + def test_adapter_custom_host_requires_exact_image(self, image_ref): + with pytest.raises(ValueError, match="exact custom-image"): + RuntimeTargetImageProfile( + provider="gcp", + image_kind="image", + image_ref=image_ref, + bootstrap_capability="preconfigured-machine-host", + management_ssh_username="host-admin", + participant_container_name="participant-desktop", + participant_username="student", + participant_readiness_contract="participant-readiness/v1", + participant_readiness_manifest_sha256="a" * 64, + ) + + def test_adapter_target_preconfigured_custom_image_is_retained(self): + runtime = RuntimeTargetImageProfile( + provider="gcp", + image_kind="image", + image_ref="projects/example/global/images/nested-host-v1", + machine_type="n2-standard-8", + disk_size_gb=220, + bootstrap_capability="preconfigured-machine-host", + management_ssh_username="host-admin", + participant_container_name="participant-desktop", + participant_username="student", + participant_readiness_contract="participant-readiness/v1", + participant_readiness_manifest_sha256="a" * 64, + ) + profile = resolve_gce_image_from_runtime_profile(_node(), runtime) + assert profile.source_image == runtime.image_ref + assert profile.source_machine_image == "" + assert profile.bootstrap_capability == "preconfigured-machine-host" + def test_adapter_target_profile_is_a_first_class_image_source(self): runtime = RuntimeTargetImageProfile( provider="gcp", diff --git a/shifter/engine/provisioner/tests/test_raes_gcp_apply.py b/shifter/engine/provisioner/tests/test_raes_gcp_apply.py index ec7e8ed1b..51396e9b6 100644 --- a/shifter/engine/provisioner/tests/test_raes_gcp_apply.py +++ b/shifter/engine/provisioner/tests/test_raes_gcp_apply.py @@ -156,6 +156,7 @@ def _apply_options( {"instance_key": instance["uuid"], "value": "virtual-machine"} for instance in plan["instances"] ], ) + overrides.setdefault("host_readiness_verifier", MagicMock()) overrides.setdefault( "operating_system_observer", lambda _plan, outputs: [ @@ -182,9 +183,9 @@ def test_missing_guest_os_evidence_fails_apply_and_cleans_up(): options = _apply_options(_config(), clients, secrets, operating_system_observer=lambda _plan, _outputs: []) with pytest.raises(ValueError, match="operating-system"): apply_raes_range_cell("req-1", 7, _plan(), _resolver, options) - # This fake reports every resource absent on readback. Cleanup must still - # revisit both guests; absent resources need no delete call. - assert clients.instances.get.call_count == 4 + # This fake reports every resource absent on readback. Cleanup revisits + # both guests for disk convergence and deletion; absent resources need no delete call. + assert clients.instances.get.call_count == 6 def test_enrolled_guest_gets_exact_broker_firewall_before_enrollment(): @@ -408,7 +409,7 @@ def test_machine_image_clone_uses_source_and_owns_every_attached_disk(self): SimpleNamespace(device_name="nested-data", auto_delete=False), ] ) - clients.instances.get.side_effect = [_NotFound(), created] + clients.instances.get.side_effect = [_NotFound(), _NotFound(), created] clients.instances.set_disk_auto_delete.return_value = SimpleNamespace(name="op") secret_ops, _ = _secret_ops() plan = _plan() @@ -433,6 +434,207 @@ def test_machine_image_clone_uses_source_and_owns_every_attached_disk(self): auto_delete=True, ) + def test_preconfigured_custom_image_uses_owned_boot_disk(self): + profile = GCERangeImageProfile( + source_image="projects/proj-1/global/images/nested-host-v1", + machine_type="n2-standard-8", + disk_size_gb=220, + bootstrap_capability=GCE_BOOTSTRAP_PRECONFIGURED_MACHINE_HOST, + participant_container_name="participant-desktop", + participant_username="operator", + host_ssh_username="hostadmin", + participant_readiness_contract="participant-readiness/v1", + participant_readiness_manifest_sha256="a" * 64, + ) + clients = _clients() + secret_ops, _ = _secret_ops() + plan = replace(_plan(), nodes=(replace(_plan().nodes[0], count=1),)) + readiness = MagicMock() + + output = apply_raes_range_cell( + "req-1", + 7, + plan, + lambda _node: profile, + _apply_options(_config(), clients, secret_ops, host_readiness_verifier=readiness), + ) + + body = clients.instances.insert.call_args.kwargs["instance_resource"] + assert body["disks"][0]["auto_delete"] is True + assert body["disks"][0]["initialize_params"]["source_image"] == profile.source_image + assert body["advanced_machine_features"] == {"enable_nested_virtualization": True} + assert output["instances"][0]["gcp_participant_container_name"] == "participant-desktop" + assert output["instances"][0]["participant_sftp_enabled"] is False + readiness.assert_called_once() + assert readiness.call_args.args[0][0]["gcp_bootstrap_capability"] == GCE_BOOTSTRAP_PRECONFIGURED_MACHINE_HOST + + def test_conflicting_preconfigured_host_is_rejected_before_mutation(self): + profile = GCERangeImageProfile( + source_image="projects/proj-1/global/images/nested-host-v1", + machine_type="n2-standard-8", + disk_size_gb=220, + bootstrap_capability=GCE_BOOTSTRAP_PRECONFIGURED_MACHINE_HOST, + participant_container_name="participant-desktop", + participant_username="operator", + host_ssh_username="hostadmin", + participant_readiness_contract="participant-readiness/v1", + participant_readiness_manifest_sha256="a" * 64, + ) + clients = _clients(exists=True) + clients.instances.get.side_effect = None + clients.instances.get.return_value = SimpleNamespace( + labels={"managed-by": "unrelated", "range-id": "other", "image-profile": "wrong"} + ) + secret_ops, secret_calls = _secret_ops() + with pytest.raises(RuntimeError, match="conflicting range or image-profile"): + apply_raes_range_cell( + "req-1", + 7, + _plan(), + lambda _node: profile, + _apply_options(_config(), clients, secret_ops), + ) + clients.instances.insert.assert_not_called() + clients.instances.delete.assert_not_called() + secret_calls.ensure_ssh.assert_not_called() + + def test_late_conflicting_host_is_never_deleted(self): + profile = GCERangeImageProfile( + source_image="projects/proj-1/global/images/nested-host-v1", + machine_type="n2-standard-8", + disk_size_gb=220, + bootstrap_capability=GCE_BOOTSTRAP_PRECONFIGURED_MACHINE_HOST, + participant_container_name="participant-desktop", + participant_username="operator", + host_ssh_username="hostadmin", + participant_readiness_contract="participant-readiness/v1", + participant_readiness_manifest_sha256="a" * 64, + ) + clients = _clients() + clients.instances.get.side_effect = [_NotFound(), SimpleNamespace(labels={"managed-by": "unrelated"})] + secret_ops, secret_calls = _secret_ops() + plan = replace(_plan(), nodes=(replace(_plan().nodes[0], count=1),)) + with pytest.raises(RuntimeError, match="conflicting range or image-profile"): + apply_raes_range_cell( + "req-1", + 7, + plan, + lambda _node: profile, + _apply_options(_config(), clients, secret_ops), + ) + clients.instances.delete.assert_not_called() + secret_calls.ensure_ssh.assert_not_called() + + def test_late_conflict_cleans_prior_created_instance_only(self): + profile = GCERangeImageProfile( + source_image="projects/proj-1/global/images/nested-host-v1", + machine_type="n2-standard-8", + disk_size_gb=220, + bootstrap_capability=GCE_BOOTSTRAP_PRECONFIGURED_MACHINE_HOST, + participant_container_name="participant-desktop", + participant_username="operator", + host_ssh_username="hostadmin", + participant_readiness_contract="participant-readiness/v1", + participant_readiness_manifest_sha256="a" * 64, + ) + raes_plan = _plan() + rendered = build_raes_range_cell_plan("req-1", 7, raes_plan, lambda _node: profile, _config()) + names = [instance["resource_name"] for instance in rendered["instances"]] + seen: dict[str, int] = {} + + def get_instance(*, instance, **_kwargs): + count = seen.get(instance, 0) + seen[instance] = count + 1 + if count == 0 or (instance == names[0] and count == 1): + raise _NotFound() + if instance == names[1]: + return SimpleNamespace(labels={"managed-by": "unrelated"}) + return SimpleNamespace(labels={}, disks=[], metadata=SimpleNamespace(items=[])) + + clients = _clients() + clients.instances.get.side_effect = get_instance + clients.networks.get.side_effect = [_NotFound(), SimpleNamespace()] + secret_ops, secret_calls = _secret_ops() + with pytest.raises(RuntimeError, match="conflicting range or image-profile"): + apply_raes_range_cell( + "req-1", + 7, + raes_plan, + lambda _node: profile, + _apply_options(_config(), clients, secret_ops), + ) + deleted = [call.kwargs["instance"] for call in clients.instances.delete.call_args_list] + assert deleted == [names[0]] + clients.networks.delete.assert_called_once() + secret_calls.delete_ssh.assert_called_once_with(7, rendered["instances"][0]["uuid"]) + + def test_custom_image_host_reconcile_restores_boot_disk_auto_delete(self): + profile = GCERangeImageProfile( + source_image="projects/proj-1/global/images/nested-host-v1", + machine_type="n2-standard-8", + disk_size_gb=220, + bootstrap_capability=GCE_BOOTSTRAP_PRECONFIGURED_MACHINE_HOST, + participant_container_name="participant-desktop", + participant_username="operator", + host_ssh_username="hostadmin", + participant_readiness_contract="participant-readiness/v1", + participant_readiness_manifest_sha256="a" * 64, + ) + raes_plan = replace(_plan(), nodes=(replace(_plan().nodes[0], count=1),)) + rendered = build_raes_range_cell_plan("req-1", 7, raes_plan, lambda _node: profile, _config()) + instance = rendered["instances"][0] + existing = SimpleNamespace( + labels={ + **rendered["labels"], + "image-key": "default", + "image-profile": instance["image_profile_fingerprint"], + }, + disks=[SimpleNamespace(device_name="boot", auto_delete=False)], + metadata=SimpleNamespace(items=[]), + ) + clients = _clients() + clients.instances.get.side_effect = None + clients.instances.get.return_value = existing + clients.instances.set_disk_auto_delete.return_value = SimpleNamespace(name="op") + secret_ops, _ = _secret_ops() + + apply_raes_range_cell( + "req-1", + 7, + raes_plan, + lambda _node: profile, + _apply_options(_config(), clients, secret_ops), + ) + + clients.instances.insert.assert_not_called() + clients.instances.set_disk_auto_delete.assert_called_once_with( + project="proj-1", + zone="us-east1-b", + instance=instance["resource_name"], + device_name="boot", + auto_delete=True, + ) + + def test_destroy_converges_retained_custom_host_boot_disk(self): + clients = _clients(exists=True) + clients.instances.get.side_effect = None + clients.instances.get.return_value = SimpleNamespace( + disks=[SimpleNamespace(device_name="boot", auto_delete=False)] + ) + clients.instances.set_disk_auto_delete.return_value = SimpleNamespace(name="op") + secret_ops, _ = _secret_ops() + raes_plan = replace(_plan(), nodes=(replace(_plan().nodes[0], count=1),)) + + destroy_raes_range_cell( + "req-1", + 7, + raes_plan, + RaesGceDestroyOptions(config=_config(), clients=clients, secret_ops=secret_ops), + ) + + clients.instances.set_disk_auto_delete.assert_called_once() + assert clients.instances.set_disk_auto_delete.call_args.kwargs["auto_delete"] is True + def test_apply_failure_triggers_cleanup_and_reraises(self): clients = _clients(instance_insert_error=RuntimeError("boom")) secret_ops, secret_mocks = _secret_ops() diff --git a/shifter/engine/provisioner/tests/test_raes_preconfigured_host_readiness.py b/shifter/engine/provisioner/tests/test_raes_preconfigured_host_readiness.py new file mode 100644 index 000000000..65a5c90f1 --- /dev/null +++ b/shifter/engine/provisioner/tests/test_raes_preconfigured_host_readiness.py @@ -0,0 +1,65 @@ +"""RAES participant-host readiness must not depend on the GCE image source kind.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from orchestrators.setup_orchestrator import SetupError +from raes_preconfigured_host_readiness import verify_preconfigured_hosts + + +def _host(source_machine_image: str = "") -> dict[str, object]: + return { + "gcp_bootstrap_capability": "preconfigured-machine-host", + "gcp_source_machine_image": source_machine_image, + "gcp_participant_container_name": "participant-desktop", + "gcp_participant_username": "student", + "gcp_participant_readiness_contract": "participant-readiness/v1", + "gcp_participant_readiness_manifest_sha256": "a" * 64, + "os": "kali", + "role": "attacker", + } + + +@pytest.mark.parametrize("source_machine_image", ["", "projects/example/global/machineImages/host-v1"]) +def test_both_host_sources_run_fixed_participant_canary(monkeypatch, source_machine_image): + execution = SimpleNamespace( + wait_for_ready=MagicMock(return_value=True), + executor=MagicMock(), + target="10.0.0.2", + document_name="shell", + close=MagicMock(), + ) + orchestrator = MagicMock() + orchestrator.orchestrate.return_value = SimpleNamespace(success=True) + monkeypatch.setattr("raes_preconfigured_host_readiness.SetupOrchestrator", lambda **_kwargs: orchestrator) + + verify_preconfigured_hosts([_host(source_machine_image)], execution_builder=lambda *_args, **_kwargs: execution) + + plan = orchestrator.orchestrate.call_args.args[1] + assert [step.name for step in plan.steps] == ["wait_for_preconfigured_machine_host", "verify_participant_readiness"] + execution.close.assert_called_once() + + +def test_host_canary_failure_blocks_readiness_and_closes_transport(monkeypatch): + execution = SimpleNamespace( + wait_for_ready=MagicMock(return_value=True), + executor=MagicMock(), + target="10.0.0.2", + document_name="shell", + close=MagicMock(), + ) + orchestrator = MagicMock() + orchestrator.orchestrate.return_value = SimpleNamespace(success=False, error="canary failed") + monkeypatch.setattr("raes_preconfigured_host_readiness.SetupOrchestrator", lambda **_kwargs: orchestrator) + + with pytest.raises(SetupError, match="participant readiness"): + verify_preconfigured_hosts([_host()], execution_builder=lambda *_args, **_kwargs: execution) + execution.close.assert_called_once() diff --git a/shifter/shifter_platform/engine/services/_raes_image.py b/shifter/shifter_platform/engine/services/_raes_image.py index 8f84d16f1..8c8ca5fba 100644 --- a/shifter/shifter_platform/engine/services/_raes_image.py +++ b/shifter/shifter_platform/engine/services/_raes_image.py @@ -294,6 +294,7 @@ def _to_view(mapping: RaesImageMapping) -> RaesImageMappingView: _MACHINE_IMAGE_REF = re.compile( r"^projects/[a-z0-9][-a-z0-9.:]*/global/machineImages/[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?$" ) +_EXACT_GCE_IMAGE_REF = re.compile(r"^projects/[a-z0-9][-a-z0-9.:]*/global/images/[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?$") _CONTAINER_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") _READINESS_CONTRACT = "participant-readiness/v1" _PRECONFIGURED_MACHINE_HOST = "preconfigured-machine-host" @@ -317,10 +318,12 @@ def _validate_runtime_profile( if not re.fullmatch(r"[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?", bootstrap): raise RaesImageMappingError("bootstrap_capability must be a lowercase logical capability") participant_fields = (container, participant_user, readiness_contract, readiness_sha) - if image_kind == "image": + if image_kind == "image" and bootstrap != _PRECONFIGURED_MACHINE_HOST: _validate_boot_image_fields(participant_fields) else: - _validate_machine_image_fields(provider, image_ref, bootstrap, management_user, participant_fields) + _validate_preconfigured_host_fields( + provider, image_ref, image_kind, bootstrap, management_user, participant_fields + ) return { "image_kind": image_kind, "bootstrap_capability": bootstrap, @@ -334,12 +337,13 @@ def _validate_runtime_profile( def _validate_boot_image_fields(participant_fields: tuple[str, str, str, str]) -> None: """Reject participant-host metadata on a normal boot image.""" if any(participant_fields): - raise RaesImageMappingError("participant host fields require image_kind 'machine-image'") + raise RaesImageMappingError("participant host fields require bootstrap_capability 'preconfigured-machine-host'") -def _validate_machine_image_fields( +def _validate_preconfigured_host_fields( provider: str, image_ref: str, + image_kind: str, bootstrap: str, management_user: str, participant_fields: tuple[str, str, str, str], @@ -347,18 +351,22 @@ def _validate_machine_image_fields( """Require the provider and complete participant-host readiness contract.""" container, participant_user, readiness_contract, readiness_sha = participant_fields if provider != "gce": - raise RaesImageMappingError("machine-image mappings currently require provider 'gce'") - if not _MACHINE_IMAGE_REF.fullmatch(image_ref): + raise RaesImageMappingError("preconfigured host mappings currently require provider 'gce'") + if image_kind == "machine-image" and not _MACHINE_IMAGE_REF.fullmatch(image_ref): raise RaesImageMappingError( "machine-image image_ref must be an exact 'projects//global/machineImages/' resource" ) + if image_kind == "image" and not _EXACT_GCE_IMAGE_REF.fullmatch(image_ref): + raise RaesImageMappingError( + "preconfigured host image_ref must be an exact 'projects//global/images/' resource" + ) if bootstrap != _PRECONFIGURED_MACHINE_HOST: raise RaesImageMappingError("machine-image mappings require bootstrap_capability 'preconfigured-machine-host'") if not management_user: - raise RaesImageMappingError("machine-image mappings require management_ssh_username") + raise RaesImageMappingError("preconfigured host mappings require management_ssh_username") if not all(participant_fields): raise RaesImageMappingError( - "machine-image mappings require participant container, username, readiness contract, and manifest digest" + "preconfigured hosts require participant container, username, readiness contract, and manifest digest" ) if not _CONTAINER_NAME.fullmatch(container): raise RaesImageMappingError("participant_container_name is invalid") diff --git a/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.test.tsx b/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.test.tsx index dc5ab45e8..a71245454 100644 --- a/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.test.tsx +++ b/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.test.tsx @@ -78,6 +78,23 @@ describe("pack adapter assignments", () => { }); }); + it("lets an administrator bind a preconfigured host from an exact custom image", async () => { + renderRoute(); + await fillAssignment(); + fireEvent.click(screen.getByLabelText("Use an administrator-selected provider image for server")); + fireEvent.change(screen.getByLabelText("Image reference for server"), { + target: { value: "projects/example/global/images/nested-host-v1" }, + }); + fireEvent.change(screen.getByLabelText("Bootstrap capability for server"), { + target: { value: "preconfigured-machine-host" }, + }); + fireEvent.change(screen.getByLabelText("Management SSH username for server"), { target: { value: "host-admin" } }); + fireEvent.change(screen.getByLabelText("Participant container for server"), { target: { value: "participant-desktop" } }); + fireEvent.change(screen.getByLabelText("Participant username for server"), { target: { value: "student" } }); + fireEvent.change(screen.getByLabelText("Readiness manifest SHA-256 for server"), { target: { value: "a".repeat(64) } }); + expect(screen.getByRole("button", { name: "Review assignment" })).toBeEnabled(); + }); + it("clears the GCP web option when switching an assignment to AWS", async () => { renderRoute(); await fillAssignment(); diff --git a/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.tsx b/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.tsx index aace7372a..1cd15b2cf 100644 --- a/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.tsx +++ b/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.tsx @@ -92,12 +92,15 @@ function validImageProfile(profile: AdapterTargetImageProfile): boolean { if (!profile.image_ref || profile.management_ssh_port < 1 || profile.management_ssh_port > 65535) return false; if (profile.provider === "aws") return !profile.allow_public_web_egress && profile.image_kind === "image" && /^ami-(?:[0-9a-f]{8}|[0-9a-f]{17})$/.test(profile.image_ref); - if (profile.image_kind === "image") { + if (profile.image_kind === "image" && profile.bootstrap_capability !== "preconfigured-machine-host") { if (profile.bootstrap_capability === "standard") return !profile.domain_dns_name && !profile.domain_netbios_name; return profile.bootstrap_capability === "prepromoted-domain-controller" && Boolean(profile.domain_dns_name && profile.domain_netbios_name); } return profile.bootstrap_capability === "preconfigured-machine-host" + && (profile.image_kind === "machine-image" + ? /^projects\/[a-z0-9][a-z0-9.:-]*\/global\/machineImages\/[a-z][-a-z0-9]*$/.test(profile.image_ref) + : /^projects\/[a-z0-9][a-z0-9.:-]*\/global\/images\/[a-z][-a-z0-9]*$/.test(profile.image_ref)) && Boolean(profile.management_ssh_username && profile.participant_container_name && profile.participant_username) && profile.participant_readiness_contract === "participant-readiness/v1" && /^[0-9a-f]{64}$/.test(profile.participant_readiness_manifest_sha256); @@ -213,7 +216,7 @@ function TargetImageProfile({ name, value, onChange }: Readonly<{ participant_container_name: "", participant_username: "", participant_readiness_manifest_sha256: "", domain_dns_name: "", domain_netbios_name: "", }); }}> - + : null} update({ bootstrap_capability: event.target.value, + participant_readiness_contract: event.target.value === "preconfigured-machine-host" ? "participant-readiness/v1" : "", + participant_container_name: "", + participant_username: "", + participant_readiness_manifest_sha256: "", domain_dns_name: "", domain_netbios_name: "", })}> + : null} {value.provider === "gcp" && value.image_kind === "image" @@ -254,7 +262,7 @@ function TargetImageProfile({ name, value, onChange }: Readonly<{ update({ domain_netbios_name })} /> : null} - {value.provider === "gcp" && value.image_kind === MACHINE_IMAGE_KIND ? <> + {value.provider === "gcp" && value.bootstrap_capability === "preconfigured-machine-host" ? <> update({ participant_container_name })} /> { )); }); + it("registers a preconfigured host from a custom image", async () => { + mockApi.mockImplementation((_path: string, options?: { method?: string }) => + Promise.resolve(options?.method === "POST" ? mapping({ image_kind: "image" }) : [])); + renderRoute(); + fireEvent.change(await screen.findByLabelText("Source name"), { target: { value: "nested-host" } }); + fireEvent.click(screen.getByLabelText("Preconfigured participant host")); + fireEvent.change(screen.getByLabelText("Image ref"), { + target: { value: "projects/example/global/images/nested-host-v1" }, + }); + fireEvent.change(screen.getByLabelText("Management SSH username"), { target: { value: "host-admin" } }); + fireEvent.change(screen.getByLabelText("Participant container"), { target: { value: "participant-desktop" } }); + fireEvent.change(screen.getByLabelText("Participant username"), { target: { value: "student" } }); + fireEvent.change(screen.getByLabelText("Readiness manifest SHA-256"), { target: { value: "a".repeat(64) } }); + fireEvent.click(screen.getByRole("button", { name: "Register mapping" })); + await waitFor(() => expect(mockApi).toHaveBeenCalledWith( + "/cms/raes-image-mappings/", + expect.objectContaining({ body: expect.objectContaining({ + image_kind: "image", + bootstrap_capability: "preconfigured-machine-host", + participant_container_name: "participant-desktop", + }) }), + )); + }); + it("disables a mapping through the API", async () => { let rows = [mapping()]; mockApi.mockImplementation( diff --git a/shifter/shifter_platform/frontend/src/features/raes-image-registry/RaesImageRegistryPage.tsx b/shifter/shifter_platform/frontend/src/features/raes-image-registry/RaesImageRegistryPage.tsx index 764d572b7..6a66e2274 100644 --- a/shifter/shifter_platform/frontend/src/features/raes-image-registry/RaesImageRegistryPage.tsx +++ b/shifter/shifter_platform/frontend/src/features/raes-image-registry/RaesImageRegistryPage.tsx @@ -92,6 +92,7 @@ function RegisterForm() { const [sourceVersion, setSourceVersion] = useState(""); const [imageRef, setImageRef] = useState(""); const [imageKind, setImageKind] = useState<"image" | "machine-image">("image"); + const [preconfiguredHost, setPreconfiguredHost] = useState(false); const [machineType, setMachineType] = useState(""); const [diskSizeGb, setDiskSizeGb] = useState(""); const [diskType, setDiskType] = useState(""); @@ -123,11 +124,11 @@ function RegisterForm() { management_ssh_port: Number(managementPort), management_ssh_username: managementUser, image_kind: imageKind, - bootstrap_capability: imageKind === MACHINE_IMAGE_KIND ? "preconfigured-machine-host" : "standard", - participant_container_name: imageKind === MACHINE_IMAGE_KIND ? participantContainer : "", - participant_username: imageKind === MACHINE_IMAGE_KIND ? participantUser : "", - participant_readiness_contract: imageKind === MACHINE_IMAGE_KIND ? readinessContract : "", - participant_readiness_manifest_sha256: imageKind === MACHINE_IMAGE_KIND ? readinessDigest : "", + bootstrap_capability: preconfiguredHost || imageKind === MACHINE_IMAGE_KIND ? "preconfigured-machine-host" : "standard", + participant_container_name: preconfiguredHost || imageKind === MACHINE_IMAGE_KIND ? participantContainer : "", + participant_username: preconfiguredHost || imageKind === MACHINE_IMAGE_KIND ? participantUser : "", + participant_readiness_contract: preconfiguredHost || imageKind === MACHINE_IMAGE_KIND ? readinessContract : "", + participant_readiness_manifest_sha256: preconfiguredHost || imageKind === MACHINE_IMAGE_KIND ? readinessDigest : "", enabled: true, notes, // This form registers a legacy alias-only mapping; portable RAES artifact @@ -146,6 +147,7 @@ function RegisterForm() { setSourceVersion(""); setImageRef(""); setImageKind("image"); + setPreconfiguredHost(false); setMachineType(""); setDiskSizeGb(""); setDiskType(""); @@ -219,6 +221,11 @@ function RegisterForm() { + {imageKind === "image" ?
+ setPreconfiguredHost(event.target.checked)} /> + +
: null}
- {imageKind === MACHINE_IMAGE_KIND ? <> + {preconfiguredHost || imageKind === MACHINE_IMAGE_KIND ? <>
None: raise RaesOperationInputError(f"{field} bootstrap_capability is invalid") if not all(isinstance(value, str) for value in participant_fields): raise RaesOperationInputError(f"{field} participant host fields are invalid") - if image_kind == "image" and any(participant_fields): - raise RaesOperationInputError(f"{field} participant host fields require a machine-image") - if image_kind == "machine-image" and not all(participant_fields): - raise RaesOperationInputError(f"{field} machine-image participant host fields are incomplete") + if bootstrap == "preconfigured-machine-host" and not all(participant_fields): + raise RaesOperationInputError(f"{field} preconfigured host fields are incomplete") + if bootstrap != "preconfigured-machine-host" and any(participant_fields): + raise RaesOperationInputError(f"{field} participant host fields require a preconfigured host") + if image_kind == "machine-image" and bootstrap != "preconfigured-machine-host": + raise RaesOperationInputError(f"{field} machine-image requires a preconfigured host") def _validated_candidate(raw: object, field: str) -> dict[str, Any]: diff --git a/shifter/shifter_platform/shared/runtime_plugin_binding.py b/shifter/shifter_platform/shared/runtime_plugin_binding.py index bbd0a5903..d0ecf0681 100644 --- a/shifter/shifter_platform/shared/runtime_plugin_binding.py +++ b/shifter/shifter_platform/shared/runtime_plugin_binding.py @@ -103,11 +103,19 @@ def _validate_gcp_image_profile(profile: RuntimeTargetImageProfile) -> None: """Validate the selected GCP boot or machine-host image profile.""" participant = _participant_profile(profile) domain = _domain_profile(profile) - if profile.image_kind == "image": + if profile.image_kind == "image" and profile.bootstrap_capability != "preconfigured-machine-host": _validate_gcp_boot_image_profile(profile, participant, domain) return - if not _GCE_MACHINE_IMAGE_REF.fullmatch(profile.image_ref): + if profile.image_kind == "machine-image" and not _GCE_MACHINE_IMAGE_REF.fullmatch(profile.image_ref): raise ValueError("GCP machine image profiles require an exact machine-image resource") + if profile.image_kind == "image" and not _GCE_IMAGE_REF.fullmatch(profile.image_ref): + raise ValueError("GCP preconfigured host profiles require an exact custom-image resource") + if ( + profile.image_kind == "image" + and profile.disk_type + and profile.disk_type not in {"pd-standard", "pd-balanced", "pd-ssd", "pd-extreme", "hyperdisk-balanced"} + ): + raise ValueError("GCP image profile disk type is unsupported") if profile.bootstrap_capability != "preconfigured-machine-host": raise ValueError("GCP machine images require the preconfigured-machine-host capability") if any(domain): @@ -131,7 +139,7 @@ def _validate_gcp_boot_image_profile( if not _GCE_IMAGE_REF.fullmatch(profile.image_ref): raise ValueError("GCP image profiles require an exact Compute Engine image resource") if any(participant): - raise ValueError("participant host fields require a GCP machine image") + raise ValueError("participant host fields require a GCP preconfigured host") if profile.bootstrap_capability == "standard" and any(domain): raise ValueError("standard GCP boot images do not accept domain fields") if profile.bootstrap_capability == "prepromoted-domain-controller" and not all(domain): diff --git a/shifter/shifter_platform/tests/engine/services/test_raes_image.py b/shifter/shifter_platform/tests/engine/services/test_raes_image.py index 2beebbf64..cfc23faf5 100644 --- a/shifter/shifter_platform/tests/engine/services/test_raes_image.py +++ b/shifter/shifter_platform/tests/engine/services/test_raes_image.py @@ -69,6 +69,44 @@ def test_normalizes_provider_case_and_whitespace(self): class TestValidation: + def test_preconfigured_host_accepts_exact_custom_image(self): + mapping = upsert_raes_image_mapping( + provider="gce", + source_name="training-host", + image_ref="projects/example/global/images/training-host-v1", + options=RaesImageMappingOptions( + image_kind="image", + bootstrap_capability="preconfigured-machine-host", + management_ssh_username="host-admin", + participant_container_name="participant-desktop", + participant_username="student", + participant_readiness_contract="participant-readiness/v1", + participant_readiness_manifest_sha256="a" * 64, + ), + ) + assert mapping.image_kind == "image" + assert mapping.bootstrap_capability == "preconfigured-machine-host" + + @pytest.mark.parametrize( + "image_ref", ["family/training-host", "projects/example/global/images/family/training-host"] + ) + def test_preconfigured_host_rejects_mutable_custom_image(self, image_ref): + with pytest.raises(RaesImageMappingError, match="exact"): + upsert_raes_image_mapping( + provider="gce", + source_name="training-host", + image_ref=image_ref, + options=RaesImageMappingOptions( + image_kind="image", + bootstrap_capability="preconfigured-machine-host", + management_ssh_username="host-admin", + participant_container_name="participant-desktop", + participant_username="student", + participant_readiness_contract="participant-readiness/v1", + participant_readiness_manifest_sha256="a" * 64, + ), + ) + def test_preconfigured_machine_host_profile_is_tenant_data(self): mapping = upsert_raes_image_mapping( provider="gce", diff --git a/shifter/shifter_platform/tests/engine/test_operation_input_raes.py b/shifter/shifter_platform/tests/engine/test_operation_input_raes.py index fbf75e4f1..0e400add2 100644 --- a/shifter/shifter_platform/tests/engine/test_operation_input_raes.py +++ b/shifter/shifter_platform/tests/engine/test_operation_input_raes.py @@ -221,6 +221,24 @@ def test_another_ranges_bindings_do_not_leak(self): class TestImageCandidates: + def test_custom_image_host_profile_crosses_as_closed_tenant_runtime_data(self): + fx = _RaesRange() + RaesImageMapping.objects.create( + provider="gce", + source_name="kali", + image_ref="projects/example/global/images/nested-host-v1", + image_kind="image", + bootstrap_capability="preconfigured-machine-host", + management_ssh_username="host-admin", + participant_container_name="participant-desktop", + participant_username="student", + participant_readiness_contract="participant-readiness/v1", + participant_readiness_manifest_sha256="a" * 64, + ) + candidate = fx.payload().image_candidates_for("gce", "kali")[0] + assert candidate.get("image_kind", "image") == "image" + assert candidate["bootstrap_capability"] == "preconfigured-machine-host" + def test_machine_host_profile_crosses_as_closed_tenant_runtime_data(self): fx = _RaesRange() RaesImageMapping.objects.create( diff --git a/shifter/shifter_platform/tests/shared/raes/test_raes_operation_input.py b/shifter/shifter_platform/tests/shared/raes/test_raes_operation_input.py index 9657b4669..d61503d0d 100644 --- a/shifter/shifter_platform/tests/shared/raes/test_raes_operation_input.py +++ b/shifter/shifter_platform/tests/shared/raes/test_raes_operation_input.py @@ -200,6 +200,24 @@ def test_plan_without_resources_yields_no_keys(self): class TestRoundTrip: + def test_preconfigured_custom_image_candidate_crosses_closed_wire(self): + payload = _built() + candidate = payload["image_candidates"]["gce:kali"][0] + candidate.update( + { + "image_ref": "projects/example/global/images/nested-host-v1", + "image_kind": "image", + "bootstrap_capability": "preconfigured-machine-host", + "participant_container_name": "participant-desktop", + "participant_username": "student", + "participant_readiness_contract": "participant-readiness/v1", + "participant_readiness_manifest_sha256": "a" * 64, + } + ) + projected = parse_raes_operation_input(payload).image_candidates_for("gce", "kali")[0] + assert projected["image_kind"] == "image" + assert projected["participant_container_name"] == "participant-desktop" + def test_parse_returns_the_built_projection(self): parsed = parse_raes_operation_input(_built()) assert parsed.plan == _plan() From 39c5ce37958a3d9ba7f4dd6126ad02243276913e Mon Sep 17 00:00:00 2001 From: Brad Edwards Date: Thu, 24 Sep 2026 05:47:53 +0000 Subject: [PATCH 2/6] refactor(gcp): split host binding and apply verification --- .../gcp_range_cell_host_binding.py | 58 ++++ shifter/engine/provisioner/gcp_range_cells.py | 61 +--- .../provisioner/raes_gcp_activation_apply.py | 2 +- shifter/engine/provisioner/raes_gcp_apply.py | 284 +++++++----------- .../provisioner/raes_gcp_attempt_cleanup.py | 68 +++++ .../provisioner/raes_gcp_verification.py | 82 +++++ .../provisioner/tests/test_raes_gcp_apply.py | 2 +- .../engine/services/_raes_image.py | 39 ++- .../administer/AdapterPackBindings.tsx | 28 +- .../shared/raes/operation_input_candidates.py | 17 +- .../shared/runtime_plugin_binding.py | 14 + 11 files changed, 381 insertions(+), 274 deletions(-) create mode 100644 shifter/engine/provisioner/gcp_range_cell_host_binding.py create mode 100644 shifter/engine/provisioner/raes_gcp_attempt_cleanup.py create mode 100644 shifter/engine/provisioner/raes_gcp_verification.py diff --git a/shifter/engine/provisioner/gcp_range_cell_host_binding.py b/shifter/engine/provisioner/gcp_range_cell_host_binding.py new file mode 100644 index 000000000..b5b54e098 --- /dev/null +++ b/shifter/engine/provisioner/gcp_range_cell_host_binding.py @@ -0,0 +1,58 @@ +"""Binding checks for deterministic GCE participant hosts.""" + +from __future__ import annotations + +from config import GCE_BOOTSTRAP_PRECONFIGURED_MACHINE_HOST +from gcp_range_cell_naming import _label_value +from gcp_range_cell_resources import HOST_PUBLIC_KEY_METADATA_KEY +from gcp_range_cell_types import InstancePlan, RangeCellPlan + + +class GCEInstanceBindingError(RuntimeError): + """An existing deterministic VM does not belong to this range profile.""" + + +def _host_public_key_from_instance(existing: object) -> str: + """Recover the provisioner-issued host key instead of minting a mismatched one.""" + metadata = getattr(existing, "metadata", None) + for item in getattr(metadata, "items", None) or []: + if getattr(item, "key", None) == HOST_PUBLIC_KEY_METADATA_KEY: + return str(getattr(item, "value", "") or "") + return "" + + +def _existing_label(existing: object, key: str) -> str: + """Read one label from a dict-like Compute instance response.""" + labels = getattr(existing, "labels", None) + getter = getattr(labels, "get", None) + if callable(getter): + return str(getter(key, "") or "") + return "" + + +def _assert_instance_image_binding(existing: object, instance: InstancePlan) -> None: + """Reject a keyed deterministic VM whose recorded profile differs from the plan.""" + expected_key = instance["image_key"] + if not expected_key: + return + actual_key = _existing_label(existing, "image-key") + actual_profile = _existing_label(existing, "image-profile") + if actual_key != expected_key or actual_profile != instance["image_profile_fingerprint"]: + raise RuntimeError( + "Existing GCE range instance has an image-profile binding that differs from the current plan; " + f"ami_key={expected_key!r}. Recreate the range instead of reusing the drifted instance." + ) + + +def _assert_preconfigured_host_binding(existing: object, plan: RangeCellPlan, instance: InstancePlan) -> None: + """Never adopt a participant host with different ownership or image policy.""" + if instance["profile"].bootstrap_capability != GCE_BOOTSTRAP_PRECONFIGURED_MACHINE_HOST: + return + expected = { + "managed-by": plan["labels"]["managed-by"], + "range-id": plan["labels"]["range-id"], + "image-key": _label_value(instance["image_key"] or "default"), + "image-profile": instance["image_profile_fingerprint"], + } + if any(_existing_label(existing, key) != value for key, value in expected.items()): + raise GCEInstanceBindingError("Existing GCE participant host has a conflicting range or image-profile binding") diff --git a/shifter/engine/provisioner/gcp_range_cells.py b/shifter/engine/provisioner/gcp_range_cells.py index a4c9f2719..e9cd4b494 100644 --- a/shifter/engine/provisioner/gcp_range_cells.py +++ b/shifter/engine/provisioner/gcp_range_cells.py @@ -15,12 +15,15 @@ _default_vertex_ops, ) from gcp_range_cell_destroy import destroy_range_cell -from gcp_range_cell_naming import _label_value +from gcp_range_cell_host_binding import ( + _assert_instance_image_binding, + _assert_preconfigured_host_binding, + _host_public_key_from_instance, +) from gcp_range_cell_ops import _get_or_none, _wait_for_operation from gcp_range_cell_outputs import InstanceCredentials, instance_output, range_cell_result, subnet_outputs from gcp_range_cell_plan import render_range_cell_plan from gcp_range_cell_resources import ( - HOST_PUBLIC_KEY_METADATA_KEY, address_resource, firewall_resource, instance_resource, @@ -52,10 +55,6 @@ logger = logging.getLogger(__name__) -class GCEInstanceBindingError(RuntimeError): - """An existing deterministic VM does not belong to this range profile.""" - - def _ensure_network(plan: RangeCellPlan, clients: GCEClients) -> bool: """Create the range VPC if it is missing.""" name = plan["network"]["name"] @@ -166,56 +165,6 @@ def _ensure_address(plan: RangeCellPlan, clients: GCEClients, instance: Instance return True -def _host_public_key_from_instance(existing: object) -> str: - """Read the provisioner-issued SSH host public key from an existing instance. - - On a reconcile the guest already serves the host key injected at create time, - so recover it from instance metadata rather than minting a mismatched one. - """ - metadata = getattr(existing, "metadata", None) - for item in getattr(metadata, "items", None) or []: - if getattr(item, "key", None) == HOST_PUBLIC_KEY_METADATA_KEY: - return str(getattr(item, "value", "") or "") - return "" - - -def _existing_label(existing: object, key: str) -> str: - """Read one label from a dict-like Compute instance response.""" - labels = getattr(existing, "labels", None) - getter = getattr(labels, "get", None) - if callable(getter): - return str(getter(key, "") or "") - return "" - - -def _assert_instance_image_binding(existing: object, instance: InstancePlan) -> None: - """Reject a keyed deterministic VM whose recorded profile differs from the plan.""" - expected_key = instance["image_key"] - if not expected_key: - return - actual_key = _existing_label(existing, "image-key") - actual_profile = _existing_label(existing, "image-profile") - if actual_key != expected_key or actual_profile != instance["image_profile_fingerprint"]: - raise RuntimeError( - "Existing GCE range instance has an image-profile binding that differs from the current plan; " - f"ami_key={expected_key!r}. Recreate the range instead of reusing the drifted instance." - ) - - -def _assert_preconfigured_host_binding(existing: object, plan: RangeCellPlan, instance: InstancePlan) -> None: - """Never adopt a deterministic participant host with different ownership or image policy.""" - if instance["profile"].bootstrap_capability != GCE_BOOTSTRAP_PRECONFIGURED_MACHINE_HOST: - return - expected = { - "managed-by": plan["labels"]["managed-by"], - "range-id": plan["labels"]["range-id"], - "image-key": _label_value(instance["image_key"] or "default"), - "image-profile": instance["image_profile_fingerprint"], - } - if any(_existing_label(existing, key) != value for key, value in expected.items()): - raise GCEInstanceBindingError("Existing GCE participant host has a conflicting range or image-profile binding") - - def _ensure_attached_disks_auto_delete( plan: RangeCellPlan, clients: GCEClients, diff --git a/shifter/engine/provisioner/raes_gcp_activation_apply.py b/shifter/engine/provisioner/raes_gcp_activation_apply.py index 23305fa6b..42b44fcd9 100644 --- a/shifter/engine/provisioner/raes_gcp_activation_apply.py +++ b/shifter/engine/provisioner/raes_gcp_activation_apply.py @@ -17,9 +17,9 @@ _bootstrap_by_node, _preflight_existing_hosts, _provision_raes_resources, - _realize_directory, ) from raes_gcp_plan import RaesGcePlanOptions, build_raes_range_cell_plan +from raes_gcp_verification import _realize_directory from raes_guest_plan import assert_management_login_separate from raes_operating_system import validate_operating_systems from raes_participant_host_keys import observe_participant_host_keys diff --git a/shifter/engine/provisioner/raes_gcp_apply.py b/shifter/engine/provisioner/raes_gcp_apply.py index bcbd0cd5f..fafcb224b 100644 --- a/shifter/engine/provisioner/raes_gcp_apply.py +++ b/shifter/engine/provisioner/raes_gcp_apply.py @@ -31,12 +31,12 @@ from config import GCERangeCellConfig, GCERangeImageProfile, load_gce_range_cell_config from gcp_range_cell_clients import GCEClients, _build_clients -from gcp_range_cell_ops import _delete_resource, _get_or_none +from gcp_range_cell_host_binding import GCEInstanceBindingError +from gcp_range_cell_ops import _get_or_none from gcp_range_cell_outputs import InstanceCredentials, instance_output, subnet_outputs from gcp_range_cell_resources import instance_resource from gcp_range_cell_types import GceEgressPolicy, InstancePlan, RangeCellPlan, ResourceDict from gcp_range_cells import ( - GCEInstanceBindingError, _assert_preconfigured_host_binding, _ensure_address, _ensure_attached_disks_auto_delete, @@ -48,10 +48,7 @@ _insert_instance, ) from raes_access import RealizedAccessBinding, join_participant_access -from raes_account_credentials import ( - default_account_credential_ops, - delete_instance_account_credentials, -) +from raes_account_credentials import default_account_credential_ops from raes_active_directory import ( default_directory_secret_ops, ) @@ -60,10 +57,12 @@ ) from raes_content_delivery import assert_content_delivery_bindings_complete from raes_gcp_apply_types import RaesGceApplyOptions, RaesGceApplyRuntime +from raes_gcp_attempt_cleanup import _cleanup_created_resources from raes_gcp_composition import node_bootstrap_script -from raes_gcp_destroy import RaesGceDestroyOptions, _instance_accounts, destroy_raes_range_cell +from raes_gcp_destroy import RaesGceDestroyOptions, destroy_raes_range_cell from raes_gcp_plan import RaesGcePlanOptions, build_raes_range_cell_plan from raes_gcp_secret_ops import RaesGceSecretOps, _default_secret_ops +from raes_gcp_verification import _verify_raes_apply from raes_guest_plan import ( _access_by_node, _accounts_by_node, @@ -71,8 +70,6 @@ _publish_participant_access, assert_management_login_separate, ) -from raes_operating_system import validate_operating_systems -from raes_participant_host_keys import observe_participant_host_keys from raes_plan import RaesPlan, RaesPlanAccount, RaesPlanNode from raes_snapshot import snapshot_resources from raes_substrate_observation import verify_prepared_source @@ -207,102 +204,64 @@ def _provision_raes_resources( for firewall in plan["firewalls"]: if _ensure_firewall(plan, runtime.clients, firewall) and created is not None: created.append(("firewall", firewall["name"])) - instance_outputs: list[ResourceDict] = [] - for instance in plan["instances"]: - if _ensure_address(plan, runtime.clients, instance) and created is not None: - created.append(("address", instance["address_name"])) - ssh_secret_ref, ssh_public_key, host_public_key = _ensure_raes_instance( - plan, - runtime.clients, - runtime.config, - instance, - runtime.secret_ops, - bootstrap_by_node, - created, - ) - output = instance_output( - plan, - instance, - InstanceCredentials( - host_ssh_secret_ref=ssh_secret_ref, - participant_ssh_secret_ref=None, - rdp_password_secret_ref=None, - ssh_public_key=ssh_public_key, - host_public_key=host_public_key, - ), - runtime.config, + return [ + _provision_raes_instance_output( + plan, runtime, instance, bootstrap_by_node, accounts_by_node, access_by_node, created ) - node_address = _node_address_of(instance) - accounts = accounts_by_node.get(node_address, ()) - account_secret_refs: dict[str, str] = {} - if accounts: - account_secret_refs = ( - runtime.credential_installer( - range_id=plan["range_id"], - instance_key=instance["uuid"], - platform=instance["os_type"], - instance_output=output, - accounts=accounts, - secret_ops=runtime.account_secret_ops, - ) - or {} - ) - _publish_participant_access(output, access_by_node.get(node_address, ()), account_secret_refs) - instance_outputs.append(output) - return instance_outputs + for instance in plan["instances"] + ] -def _cleanup_created_resources( +def _provision_raes_instance_output( plan: RangeCellPlan, - raes_plan: RaesPlan, runtime: RaesGceApplyRuntime, - created: list[tuple[str, str]], -) -> None: - """Release only this attempt's resources after a late foreign-VM conflict.""" - clients = runtime.clients - services = { - "instance": (clients.instances, "zone", "instance", {"zone": plan["zone"]}), - "address": (clients.addresses, "region", "address", {"region": plan["region"]}), - "firewall": (clients.firewalls, "global", "firewall", {}), - "router": (clients.routers, "region", "router", {"region": plan["region"]}), - "subnetwork": (clients.subnetworks, "region", "subnetwork", {"region": plan["region"]}), - "network": (clients.networks, "global", "network", {}), - } - instances = {instance["resource_name"]: instance for instance in plan["instances"]} - for kind, name in reversed(created): - service, scope, field, extra = services[kind] - try: - if kind == "instance": - existing = _get_or_none( - clients.instances.get, - clients.google_exceptions, - project=plan["project_id"], - zone=plan["zone"], - instance=name, - ) - if existing is not None: - _ensure_attached_disks_auto_delete(plan, clients, name, existing) - _delete_resource( - plan, - clients, - service.get, - service.delete, - scope, - project=plan["project_id"], - **extra, - **{field: name}, + instance: InstancePlan, + bootstrap_by_node: dict[str, str], + accounts_by_node: dict[str, tuple[RaesPlanAccount, ...]], + access_by_node: dict[str, tuple[RealizedAccessBinding, ...]], + created: list[tuple[str, str]] | None, +) -> ResourceDict: + """Provision one RAES host and publish only realized participant access.""" + if _ensure_address(plan, runtime.clients, instance) and created is not None: + created.append(("address", instance["address_name"])) + ssh_secret_ref, ssh_public_key, host_public_key = _ensure_raes_instance( + plan, + runtime.clients, + runtime.config, + instance, + runtime.secret_ops, + bootstrap_by_node, + created, + ) + output = instance_output( + plan, + instance, + InstanceCredentials( + host_ssh_secret_ref=ssh_secret_ref, + participant_ssh_secret_ref=None, + rdp_password_secret_ref=None, + ssh_public_key=ssh_public_key, + host_public_key=host_public_key, + ), + runtime.config, + ) + node_address = _node_address_of(instance) + accounts = accounts_by_node.get(node_address, ()) + account_secret_refs: dict[str, str] = {} + if accounts: + account_secret_refs = ( + runtime.credential_installer( + range_id=plan["range_id"], + instance_key=instance["uuid"], + platform=instance["os_type"], + instance_output=output, + accounts=accounts, + secret_ops=runtime.account_secret_ops, ) - if kind == "instance": - instance = instances[name] - runtime.secret_ops.delete_ssh(plan["range_id"], instance["uuid"]) - delete_instance_account_credentials( - plan["range_id"], - instance["uuid"], - _instance_accounts(raes_plan, instance), - runtime.account_secret_ops, - ) - except Exception as exc: - logger.error("Failed to clean attempt-created GCE resource kind=%s error_type=%s", kind, type(exc).__name__) + or {} + ) + _publish_participant_access(output, access_by_node.get(node_address, ()), account_secret_refs) + return output def _preflight_existing_hosts(plan: RangeCellPlan, clients: GCEClients) -> None: @@ -326,48 +285,6 @@ def _bootstrap_by_node(raes_plan: RaesPlan) -> dict[str, str]: return {node.address: script for node in raes_plan.nodes if (script := node_bootstrap_script(node, raes_plan))} -def _realize_directory( - plan: RangeCellPlan, - raes_plan: RaesPlan, - instance_outputs: list[ResourceDict], - runtime: RaesGceApplyRuntime, -) -> frozenset[str]: - """Realize admitted directory topology when the plan carries a domain.""" - if raes_plan.domains: - runtime.directory_realizer( - range_id=plan["range_id"], - raes_plan=raes_plan, - instance_outputs=instance_outputs, - secret_ops=runtime.directory_secret_ops, - ) - return frozenset( - account.address - for account in raes_plan.accounts - if account.domain_ref is not None or account.domain_id is not None - ) - return frozenset() - - -def _realize_content_delivery( - raes_plan: RaesPlan, - instance_outputs: list[ResourceDict], - delivery_bindings: list[dict[str, Any]] | None, - runtime: RaesGceApplyRuntime, -) -> frozenset[str]: - """Deliver every source-backed content item when the plan carries one (#1564).""" - if any(item.source_name for item in raes_plan.content) or bool(raes_plan.features): - runtime.content_delivery_realizer( - raes_plan=raes_plan, - instance_outputs=instance_outputs, - delivery_bindings=delivery_bindings, - ) - return frozenset( - [item.address for item in raes_plan.content if item.source_name] - + [feature.address for feature in raes_plan.features] - ) - return frozenset() - - def _cleanup_failed_apply( request_uuid: str, range_id: int, @@ -390,6 +307,44 @@ def _cleanup_failed_apply( ) +def _prepare_raes_apply( + request_uuid: str, + range_id: int, + raes_plan: RaesPlan, + resolve_image: Callable[[RaesPlanNode], GCERangeImageProfile], + options: RaesGceApplyOptions, + config: GCERangeCellConfig, + delivery_bindings: list[dict[str, Any]] | None, + access_bindings: list[dict[str, Any]] | None, +) -> tuple[RangeCellPlan, tuple[RealizedAccessBinding, ...]]: + """Validate and plan the complete range before provider mutation.""" + realized_access = join_participant_access(access_bindings or (), raes_plan) + _assert_composition_targets_resolve(raes_plan) + _assert_content_delivery_bindings_complete(raes_plan, delivery_bindings) + assert_composition_is_verifiable(raes_plan) + expected_composition = { + *[item.address for item in raes_plan.content], + *[account.address for account in raes_plan.accounts], + *[feature.address for feature in raes_plan.features], + } + snapshot_resources(raes_plan, expected_composition) + plan = build_raes_range_cell_plan( + request_uuid, + range_id, + raes_plan, + resolve_image, + RaesGcePlanOptions( + config=config, + access_bindings=realized_access, + egress_policy=GceEgressPolicy(mode=options.egress_mode, model_broker=options.model_broker), + allocated_network_cidrs=options.allocated_network_cidrs, + ), + ) + for instance in plan["instances"]: + assert_management_login_separate(raes_plan, _node_address_of(instance), instance["host_ssh_username"]) + return plan, realized_access + + def apply_raes_range_cell( request_uuid: str, range_id: int, @@ -418,33 +373,10 @@ def apply_raes_range_cell( created: list[tuple[str, str]] = [] plan: RangeCellPlan | None = None try: - realized_access = join_participant_access(access_bindings or (), raes_plan) - _assert_composition_targets_resolve(raes_plan) - _assert_content_delivery_bindings_complete(raes_plan, delivery_bindings) - assert_composition_is_verifiable(raes_plan) - expected_composition = { - *[item.address for item in raes_plan.content], - *[account.address for account in raes_plan.accounts], - *[feature.address for feature in raes_plan.features], - } - # Build and size-check the complete sanitized evidence shape before cloud mutation. - snapshot_resources(raes_plan, expected_composition) - plan = build_raes_range_cell_plan( - request_uuid, - range_id, - raes_plan, - resolve_image, - RaesGcePlanOptions( - config=resolved_config, - access_bindings=realized_access, - egress_policy=GceEgressPolicy( - mode=resolved_options.egress_mode, model_broker=resolved_options.model_broker - ), - allocated_network_cidrs=resolved_options.allocated_network_cidrs, - ), + plan, realized_access = _prepare_raes_apply( + request_uuid, range_id, raes_plan, resolve_image, resolved_options, resolved_config, + delivery_bindings, access_bindings, ) - for instance in plan["instances"]: - assert_management_login_separate(raes_plan, _node_address_of(instance), instance["host_ssh_username"]) runtime = _apply_runtime(resolved_options, config=resolved_config) _preflight_existing_hosts(plan, runtime.clients) mutation_started = True @@ -456,19 +388,7 @@ def apply_raes_range_cell( _access_by_node(realized_access), created, ) - verified = set(_realize_directory(plan, raes_plan, instance_outputs, runtime)) - verified.update(_realize_content_delivery(raes_plan, instance_outputs, delivery_bindings, runtime)) - if runtime.model_enrollment is not None: - runtime.model_enrollment(raes_plan, instance_outputs) - if runtime.runtime_plugin is not None: - runtime.runtime_plugin(raes_plan, instance_outputs) - runtime.host_readiness_verifier(instance_outputs) - observe_participant_host_keys(instance_outputs) - verified.update(runtime.composition_verifier(raes_plan, instance_outputs)) - operating_systems = runtime.operating_system_observer(raes_plan, instance_outputs) - validate_operating_systems(raes_plan, operating_systems) - compute_substrates = runtime.substrate_observer(plan, runtime.clients) - snapshot_resources(raes_plan, verified) + verified_observations = _verify_raes_apply(plan, raes_plan, instance_outputs, delivery_bindings, runtime) except GCEInstanceBindingError: # A conflicting VM is not ours to delete, even if a race placed it # after the read-only preflight and some network resources were made. @@ -490,9 +410,7 @@ def apply_raes_range_cell( return { "subnets": subnet_outputs(plan), "instances": instance_outputs, - "composition_verified_addresses": sorted(verified), - "operating_systems": operating_systems, - "compute_substrates": compute_substrates, + **verified_observations, } diff --git a/shifter/engine/provisioner/raes_gcp_attempt_cleanup.py b/shifter/engine/provisioner/raes_gcp_attempt_cleanup.py new file mode 100644 index 000000000..8e68d2ba8 --- /dev/null +++ b/shifter/engine/provisioner/raes_gcp_attempt_cleanup.py @@ -0,0 +1,68 @@ +"""Cleanup of only the GCE resources created during a failed RAES apply attempt.""" + +from __future__ import annotations + +import logging + +from gcp_range_cell_ops import _delete_resource, _get_or_none +from gcp_range_cell_types import RangeCellPlan +from gcp_range_cells import _ensure_attached_disks_auto_delete +from raes_account_credentials import delete_instance_account_credentials +from raes_gcp_apply_types import RaesGceApplyRuntime +from raes_gcp_destroy import _instance_accounts +from raes_plan import RaesPlan + +logger = logging.getLogger(__name__) + + +def _cleanup_created_resources( + plan: RangeCellPlan, + raes_plan: RaesPlan, + runtime: RaesGceApplyRuntime, + created: list[tuple[str, str]], +) -> None: + """Release only this attempt's resources after a late foreign-VM conflict.""" + clients = runtime.clients + services = { + "instance": (clients.instances, "zone", "instance", {"zone": plan["zone"]}), + "address": (clients.addresses, "region", "address", {"region": plan["region"]}), + "firewall": (clients.firewalls, "global", "firewall", {}), + "router": (clients.routers, "region", "router", {"region": plan["region"]}), + "subnetwork": (clients.subnetworks, "region", "subnetwork", {"region": plan["region"]}), + "network": (clients.networks, "global", "network", {}), + } + instances = {instance["resource_name"]: instance for instance in plan["instances"]} + for kind, name in reversed(created): + service, scope, field, extra = services[kind] + try: + if kind == "instance": + existing = _get_or_none( + clients.instances.get, + clients.google_exceptions, + project=plan["project_id"], + zone=plan["zone"], + instance=name, + ) + if existing is not None: + _ensure_attached_disks_auto_delete(plan, clients, name, existing) + _delete_resource( + plan, + clients, + service.get, + service.delete, + scope, + project=plan["project_id"], + **extra, + **{field: name}, + ) + if kind == "instance": + instance = instances[name] + runtime.secret_ops.delete_ssh(plan["range_id"], instance["uuid"]) + delete_instance_account_credentials( + plan["range_id"], + instance["uuid"], + _instance_accounts(raes_plan, instance), + runtime.account_secret_ops, + ) + except Exception: + logger.exception("Failed to clean attempt-created GCE resource kind=%s", kind) diff --git a/shifter/engine/provisioner/raes_gcp_verification.py b/shifter/engine/provisioner/raes_gcp_verification.py new file mode 100644 index 000000000..772e87dd0 --- /dev/null +++ b/shifter/engine/provisioner/raes_gcp_verification.py @@ -0,0 +1,82 @@ +"""Post-provisioning RAES guest realization and evidence checks on GCE.""" + +from __future__ import annotations + +from typing import Any + +from gcp_range_cell_types import RangeCellPlan, ResourceDict +from raes_gcp_apply_types import RaesGceApplyRuntime +from raes_operating_system import validate_operating_systems +from raes_participant_host_keys import observe_participant_host_keys +from raes_plan import RaesPlan +from raes_snapshot import snapshot_resources + + +def _realize_directory( + plan: RangeCellPlan, + raes_plan: RaesPlan, + instance_outputs: list[ResourceDict], + runtime: RaesGceApplyRuntime, +) -> frozenset[str]: + """Realize admitted directory topology when the plan carries a domain.""" + if raes_plan.domains: + runtime.directory_realizer( + range_id=plan["range_id"], + raes_plan=raes_plan, + instance_outputs=instance_outputs, + secret_ops=runtime.directory_secret_ops, + ) + return frozenset( + account.address + for account in raes_plan.accounts + if account.domain_ref is not None or account.domain_id is not None + ) + return frozenset() + + +def _realize_content_delivery( + raes_plan: RaesPlan, + instance_outputs: list[ResourceDict], + delivery_bindings: list[dict[str, Any]] | None, + runtime: RaesGceApplyRuntime, +) -> frozenset[str]: + """Deliver every source-backed content item when the plan carries one.""" + if any(item.source_name for item in raes_plan.content) or bool(raes_plan.features): + runtime.content_delivery_realizer( + raes_plan=raes_plan, + instance_outputs=instance_outputs, + delivery_bindings=delivery_bindings, + ) + return frozenset( + [item.address for item in raes_plan.content if item.source_name] + + [feature.address for feature in raes_plan.features] + ) + return frozenset() + + +def _verify_raes_apply( + plan: RangeCellPlan, + raes_plan: RaesPlan, + instance_outputs: list[ResourceDict], + delivery_bindings: list[dict[str, Any]] | None, + runtime: RaesGceApplyRuntime, +) -> ResourceDict: + """Complete guest realization and return verified range observations.""" + verified = set(_realize_directory(plan, raes_plan, instance_outputs, runtime)) + verified.update(_realize_content_delivery(raes_plan, instance_outputs, delivery_bindings, runtime)) + if runtime.model_enrollment is not None: + runtime.model_enrollment(raes_plan, instance_outputs) + if runtime.runtime_plugin is not None: + runtime.runtime_plugin(raes_plan, instance_outputs) + runtime.host_readiness_verifier(instance_outputs) + observe_participant_host_keys(instance_outputs) + verified.update(runtime.composition_verifier(raes_plan, instance_outputs)) + operating_systems = runtime.operating_system_observer(raes_plan, instance_outputs) + validate_operating_systems(raes_plan, operating_systems) + compute_substrates = runtime.substrate_observer(plan, runtime.clients) + snapshot_resources(raes_plan, verified) + return { + "composition_verified_addresses": sorted(verified), + "operating_systems": operating_systems, + "compute_substrates": compute_substrates, + } diff --git a/shifter/engine/provisioner/tests/test_raes_gcp_apply.py b/shifter/engine/provisioner/tests/test_raes_gcp_apply.py index 51396e9b6..b3b37d5b3 100644 --- a/shifter/engine/provisioner/tests/test_raes_gcp_apply.py +++ b/shifter/engine/provisioner/tests/test_raes_gcp_apply.py @@ -1414,7 +1414,7 @@ def test_participant_identity_readback_failure_prevents_readiness(self, monkeypa def observe(instances): observe_participant_host_keys(instances, execution_builder=lambda *_args, **_kwargs: context) - module = "raes_gcp_activation_apply" if warm else "raes_gcp_apply" + module = "raes_gcp_activation_apply" if warm else "raes_gcp_verification" monkeypatch.setattr(f"{module}.observe_participant_host_keys", observe, raising=False) apply = realize_access_on_existing_cell if warm else apply_raes_range_cell diff --git a/shifter/shifter_platform/engine/services/_raes_image.py b/shifter/shifter_platform/engine/services/_raes_image.py index 8c8ca5fba..43ae690fd 100644 --- a/shifter/shifter_platform/engine/services/_raes_image.py +++ b/shifter/shifter_platform/engine/services/_raes_image.py @@ -307,16 +307,11 @@ def _validate_runtime_profile( management_user: str, ) -> dict[str, str]: """Validate the optional provider realization profile stored with a mapping.""" - image_kind = (opts.image_kind or "image").strip() - bootstrap = (opts.bootstrap_capability or "standard").strip() + image_kind, bootstrap = _validated_profile_discriminator(opts) container = (opts.participant_container_name or "").strip() participant_user = (opts.participant_username or "").strip() readiness_contract = (opts.participant_readiness_contract or "").strip() readiness_sha = (opts.participant_readiness_manifest_sha256 or "").strip() - if image_kind not in {"image", "machine-image"}: - raise RaesImageMappingError("image_kind must be 'image' or 'machine-image'") - if not re.fullmatch(r"[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?", bootstrap): - raise RaesImageMappingError("bootstrap_capability must be a lowercase logical capability") participant_fields = (container, participant_user, readiness_contract, readiness_sha) if image_kind == "image" and bootstrap != _PRECONFIGURED_MACHINE_HOST: _validate_boot_image_fields(participant_fields) @@ -334,6 +329,17 @@ def _validate_runtime_profile( } +def _validated_profile_discriminator(opts: RaesImageMappingOptions) -> tuple[str, str]: + """Validate the image kind and bootstrap capability before profile details.""" + image_kind = (opts.image_kind or "image").strip() + bootstrap = (opts.bootstrap_capability or "standard").strip() + if image_kind not in {"image", "machine-image"}: + raise RaesImageMappingError("image_kind must be 'image' or 'machine-image'") + if not re.fullmatch(r"[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?", bootstrap): + raise RaesImageMappingError("bootstrap_capability must be a lowercase logical capability") + return image_kind, bootstrap + + def _validate_boot_image_fields(participant_fields: tuple[str, str, str, str]) -> None: """Reject participant-host metadata on a normal boot image.""" if any(participant_fields): @@ -352,14 +358,7 @@ def _validate_preconfigured_host_fields( container, participant_user, readiness_contract, readiness_sha = participant_fields if provider != "gce": raise RaesImageMappingError("preconfigured host mappings currently require provider 'gce'") - if image_kind == "machine-image" and not _MACHINE_IMAGE_REF.fullmatch(image_ref): - raise RaesImageMappingError( - "machine-image image_ref must be an exact 'projects//global/machineImages/' resource" - ) - if image_kind == "image" and not _EXACT_GCE_IMAGE_REF.fullmatch(image_ref): - raise RaesImageMappingError( - "preconfigured host image_ref must be an exact 'projects//global/images/' resource" - ) + _validate_preconfigured_image_ref(image_ref, image_kind) if bootstrap != _PRECONFIGURED_MACHINE_HOST: raise RaesImageMappingError("machine-image mappings require bootstrap_capability 'preconfigured-machine-host'") if not management_user: @@ -380,6 +379,18 @@ def _validate_preconfigured_host_fields( raise RaesImageMappingError("participant_readiness_manifest_sha256 must be a lowercase SHA-256 digest") +def _validate_preconfigured_image_ref(image_ref: str, image_kind: str) -> None: + """Require an exact source for either supported preconfigured host image kind.""" + if image_kind == "machine-image" and not _MACHINE_IMAGE_REF.fullmatch(image_ref): + raise RaesImageMappingError( + "machine-image image_ref must be an exact 'projects//global/machineImages/' resource" + ) + if image_kind == "image" and not _EXACT_GCE_IMAGE_REF.fullmatch(image_ref): + raise RaesImageMappingError( + "preconfigured host image_ref must be an exact 'projects//global/images/' resource" + ) + + def _stripped(value: str | None) -> str: """Return ``value`` stripped, or ``""`` when it is None/blank.""" return (value or "").strip() diff --git a/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.tsx b/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.tsx index 1cd15b2cf..3661adf0a 100644 --- a/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.tsx +++ b/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.tsx @@ -21,6 +21,8 @@ import { manifestPreview } from "./adapter-manifest"; import { PackUploadForm } from "./PackUploadForm"; const MACHINE_IMAGE_KIND = "machine-image"; +const BOOT_IMAGE_KIND = "image"; +const PRECONFIGURED_HOST_CAPABILITY = "preconfigured-machine-host"; function bindingStatus(pack: AdapterPack): string { if (!pack.binding) return "No adapter assigned"; @@ -71,7 +73,7 @@ function declaredNames(value: unknown, key: string): string[] { const emptyImageProfile = (): AdapterTargetImageProfile => ({ provider: "gcp", - image_kind: "image", + image_kind: BOOT_IMAGE_KIND, image_ref: "", machine_type: "", disk_size_gb: null, @@ -91,13 +93,13 @@ const emptyImageProfile = (): AdapterTargetImageProfile => ({ function validImageProfile(profile: AdapterTargetImageProfile): boolean { if (!profile.image_ref || profile.management_ssh_port < 1 || profile.management_ssh_port > 65535) return false; if (profile.provider === "aws") return !profile.allow_public_web_egress - && profile.image_kind === "image" && /^ami-(?:[0-9a-f]{8}|[0-9a-f]{17})$/.test(profile.image_ref); - if (profile.image_kind === "image" && profile.bootstrap_capability !== "preconfigured-machine-host") { + && profile.image_kind === BOOT_IMAGE_KIND && /^ami-(?:[0-9a-f]{8}|[0-9a-f]{17})$/.test(profile.image_ref); + if (profile.image_kind === BOOT_IMAGE_KIND && profile.bootstrap_capability !== PRECONFIGURED_HOST_CAPABILITY) { if (profile.bootstrap_capability === "standard") return !profile.domain_dns_name && !profile.domain_netbios_name; return profile.bootstrap_capability === "prepromoted-domain-controller" && Boolean(profile.domain_dns_name && profile.domain_netbios_name); } - return profile.bootstrap_capability === "preconfigured-machine-host" + return profile.bootstrap_capability === PRECONFIGURED_HOST_CAPABILITY && (profile.image_kind === "machine-image" ? /^projects\/[a-z0-9][a-z0-9.:-]*\/global\/machineImages\/[a-z][-a-z0-9]*$/.test(profile.image_ref) : /^projects\/[a-z0-9][a-z0-9.:-]*\/global\/images\/[a-z][-a-z0-9]*$/.test(profile.image_ref)) @@ -199,7 +201,7 @@ function TargetImageProfile({ name, value, onChange }: Readonly<{
{ const machine = event.target.value === MACHINE_IMAGE_KIND; update({ - image_kind: machine ? MACHINE_IMAGE_KIND : "image", - bootstrap_capability: machine ? "preconfigured-machine-host" : "standard", + image_kind: machine ? MACHINE_IMAGE_KIND : BOOT_IMAGE_KIND, + bootstrap_capability: machine ? PRECONFIGURED_HOST_CAPABILITY : "standard", participant_readiness_contract: machine ? "participant-readiness/v1" : "", participant_container_name: "", participant_username: "", participant_readiness_manifest_sha256: "", domain_dns_name: "", domain_netbios_name: "", }); }}> - +
: null} update({ allow_public_web_egress: event.target.checked })} />
: null} - {value.provider === "gcp" && value.image_kind === "image" ?
+ {value.provider === "gcp" && value.image_kind === BOOT_IMAGE_KIND ?
: null} - {value.provider === "gcp" && value.image_kind === "image" + {value.provider === "gcp" && value.image_kind === BOOT_IMAGE_KIND && value.bootstrap_capability === "prepromoted-domain-controller" ? <> update({ domain_dns_name })} /> update({ domain_netbios_name })} /> : null} - {value.provider === "gcp" && value.bootstrap_capability === "preconfigured-machine-host" ? <> + {value.provider === "gcp" && value.bootstrap_capability === PRECONFIGURED_HOST_CAPABILITY ? <> update({ participant_container_name })} /> None: """Require a complete host contract only for a machine-image candidate.""" image_kind = candidate.get("image_kind", "image") bootstrap = candidate.get("bootstrap_capability", "standard") + if image_kind not in {"image", "machine-image"}: + raise RaesOperationInputError(f"{field} image_kind is invalid") + if not isinstance(bootstrap, str) or not bootstrap: + raise RaesOperationInputError(f"{field} bootstrap_capability is invalid") + _validate_participant_host_fields(candidate, field, bootstrap) + if image_kind == "machine-image" and bootstrap != "preconfigured-machine-host": + raise RaesOperationInputError(f"{field} machine-image requires a preconfigured host") + + +def _validate_participant_host_fields(candidate: dict[str, Any], field: str, bootstrap: str) -> None: + """Validate the participant-facing portion of a candidate host contract.""" participant_fields = ( candidate.get("participant_container_name", ""), candidate.get("participant_username", ""), candidate.get("participant_readiness_contract", ""), candidate.get("participant_readiness_manifest_sha256", ""), ) - if image_kind not in {"image", "machine-image"}: - raise RaesOperationInputError(f"{field} image_kind is invalid") - if not isinstance(bootstrap, str) or not bootstrap: - raise RaesOperationInputError(f"{field} bootstrap_capability is invalid") if not all(isinstance(value, str) for value in participant_fields): raise RaesOperationInputError(f"{field} participant host fields are invalid") if bootstrap == "preconfigured-machine-host" and not all(participant_fields): raise RaesOperationInputError(f"{field} preconfigured host fields are incomplete") if bootstrap != "preconfigured-machine-host" and any(participant_fields): raise RaesOperationInputError(f"{field} participant host fields require a preconfigured host") - if image_kind == "machine-image" and bootstrap != "preconfigured-machine-host": - raise RaesOperationInputError(f"{field} machine-image requires a preconfigured host") def _validated_candidate(raw: object, field: str) -> dict[str, Any]: diff --git a/shifter/shifter_platform/shared/runtime_plugin_binding.py b/shifter/shifter_platform/shared/runtime_plugin_binding.py index d0ecf0681..cd2622937 100644 --- a/shifter/shifter_platform/shared/runtime_plugin_binding.py +++ b/shifter/shifter_platform/shared/runtime_plugin_binding.py @@ -106,6 +106,12 @@ def _validate_gcp_image_profile(profile: RuntimeTargetImageProfile) -> None: if profile.image_kind == "image" and profile.bootstrap_capability != "preconfigured-machine-host": _validate_gcp_boot_image_profile(profile, participant, domain) return + _validate_gcp_preconfigured_source(profile) + _validate_gcp_preconfigured_readiness(profile, participant, domain) + + +def _validate_gcp_preconfigured_source(profile: RuntimeTargetImageProfile) -> None: + """Require an exact GCP host source and supported custom-image disk type.""" if profile.image_kind == "machine-image" and not _GCE_MACHINE_IMAGE_REF.fullmatch(profile.image_ref): raise ValueError("GCP machine image profiles require an exact machine-image resource") if profile.image_kind == "image" and not _GCE_IMAGE_REF.fullmatch(profile.image_ref): @@ -116,6 +122,14 @@ def _validate_gcp_image_profile(profile: RuntimeTargetImageProfile) -> None: and profile.disk_type not in {"pd-standard", "pd-balanced", "pd-ssd", "pd-extreme", "hyperdisk-balanced"} ): raise ValueError("GCP image profile disk type is unsupported") + + +def _validate_gcp_preconfigured_readiness( + profile: RuntimeTargetImageProfile, + participant: tuple[str, str, str, str], + domain: tuple[str, str], +) -> None: + """Require a complete participant access and readiness contract.""" if profile.bootstrap_capability != "preconfigured-machine-host": raise ValueError("GCP machine images require the preconfigured-machine-host capability") if any(domain): From 9dfd8320e8808562dfc048d4dc63b55c35142101 Mon Sep 17 00:00:00 2001 From: Brad Edwards Date: Thu, 24 Sep 2026 05:50:10 +0000 Subject: [PATCH 3/6] style(gcp): format apply orchestration --- shifter/engine/provisioner/raes_gcp_apply.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/shifter/engine/provisioner/raes_gcp_apply.py b/shifter/engine/provisioner/raes_gcp_apply.py index fafcb224b..e9ce888bd 100644 --- a/shifter/engine/provisioner/raes_gcp_apply.py +++ b/shifter/engine/provisioner/raes_gcp_apply.py @@ -374,8 +374,14 @@ def apply_raes_range_cell( plan: RangeCellPlan | None = None try: plan, realized_access = _prepare_raes_apply( - request_uuid, range_id, raes_plan, resolve_image, resolved_options, resolved_config, - delivery_bindings, access_bindings, + request_uuid, + range_id, + raes_plan, + resolve_image, + resolved_options, + resolved_config, + delivery_bindings, + access_bindings, ) runtime = _apply_runtime(resolved_options, config=resolved_config) _preflight_existing_hosts(plan, runtime.clients) From ab1a5d2a5b3be911e7800a50121684baf19fd342 Mon Sep 17 00:00:00 2001 From: Brad Edwards Date: Thu, 24 Sep 2026 06:13:05 +0000 Subject: [PATCH 4/6] refactor(gcp): simplify range apply orchestration --- shifter/engine/provisioner/raes_gcp_apply.py | 87 +++++++++++++------ .../administer/AdapterPackBindings.tsx | 2 +- 2 files changed, 61 insertions(+), 28 deletions(-) diff --git a/shifter/engine/provisioner/raes_gcp_apply.py b/shifter/engine/provisioner/raes_gcp_apply.py index e9ce888bd..80f7e78c9 100644 --- a/shifter/engine/provisioner/raes_gcp_apply.py +++ b/shifter/engine/provisioner/raes_gcp_apply.py @@ -27,6 +27,7 @@ import base64 import logging from collections.abc import Callable +from dataclasses import dataclass from typing import Any from config import GCERangeCellConfig, GCERangeImageProfile, load_gce_range_cell_config @@ -78,6 +79,17 @@ logger = logging.getLogger(__name__) +@dataclass(frozen=True, slots=True) +class _ApplyBindings: + delivery: list[dict[str, Any]] | None + access: list[dict[str, Any]] | None + + +def _record_created(created: list[tuple[str, str]] | None, kind: str, name: str, was_created: bool) -> None: + if was_created and created is not None: + created.append((kind, name)) + + def _apply_runtime( options: RaesGceApplyOptions, *, @@ -194,16 +206,16 @@ def _provision_raes_resources( authored account credential installed and verified on the guest, so a declared endpoint never appears with a credential that was never realized. """ - if plan["manage_network"] and _ensure_network(plan, runtime.clients) and created is not None: - created.append(("network", plan["network"]["name"])) + if plan["manage_network"]: + _record_created(created, "network", plan["network"]["name"], _ensure_network(plan, runtime.clients)) for subnet in plan["subnets"]: - if _ensure_subnetwork(plan, runtime.clients, subnet) and created is not None: - created.append(("subnetwork", subnet["resource_name"])) - if _ensure_router_nat(plan, runtime.clients) and created is not None: - created.append(("router", plan["router_nat"]["router_name"])) + _record_created( + created, "subnetwork", subnet["resource_name"], _ensure_subnetwork(plan, runtime.clients, subnet) + ) + if router_nat := plan.get("router_nat"): + _record_created(created, "router", router_nat["router_name"], _ensure_router_nat(plan, runtime.clients)) for firewall in plan["firewalls"]: - if _ensure_firewall(plan, runtime.clients, firewall) and created is not None: - created.append(("firewall", firewall["name"])) + _record_created(created, "firewall", firewall["name"], _ensure_firewall(plan, runtime.clients, firewall)) return [ _provision_raes_instance_output( plan, runtime, instance, bootstrap_by_node, accounts_by_node, access_by_node, created @@ -222,8 +234,7 @@ def _provision_raes_instance_output( created: list[tuple[str, str]] | None, ) -> ResourceDict: """Provision one RAES host and publish only realized participant access.""" - if _ensure_address(plan, runtime.clients, instance) and created is not None: - created.append(("address", instance["address_name"])) + _record_created(created, "address", instance["address_name"], _ensure_address(plan, runtime.clients, instance)) ssh_secret_ref, ssh_public_key, host_public_key = _ensure_raes_instance( plan, runtime.clients, @@ -314,13 +325,12 @@ def _prepare_raes_apply( resolve_image: Callable[[RaesPlanNode], GCERangeImageProfile], options: RaesGceApplyOptions, config: GCERangeCellConfig, - delivery_bindings: list[dict[str, Any]] | None, - access_bindings: list[dict[str, Any]] | None, + bindings: _ApplyBindings, ) -> tuple[RangeCellPlan, tuple[RealizedAccessBinding, ...]]: """Validate and plan the complete range before provider mutation.""" - realized_access = join_participant_access(access_bindings or (), raes_plan) + realized_access = join_participant_access(bindings.access or (), raes_plan) _assert_composition_targets_resolve(raes_plan) - _assert_content_delivery_bindings_complete(raes_plan, delivery_bindings) + _assert_content_delivery_bindings_complete(raes_plan, bindings.delivery) assert_composition_is_verifiable(raes_plan) expected_composition = { *[item.address for item in raes_plan.content], @@ -345,6 +355,39 @@ def _prepare_raes_apply( return plan, realized_access +def _cleanup_conflicting_apply( + plan: RangeCellPlan | None, + raes_plan: RaesPlan, + runtime: RaesGceApplyRuntime | None, + created: list[tuple[str, str]], + mutation_started: bool, + options: RaesGceApplyOptions, +) -> None: + """Keep a conflicting deterministic VM while removing only this attempt's resources.""" + if mutation_started and runtime is not None and plan is not None: + _cleanup_created_resources(plan, raes_plan, runtime, created) + if not mutation_started and options.on_pre_mutation_failure is not None: + options.on_pre_mutation_failure() + + +def _cleanup_failed_apply_attempt( + request_uuid: str, + range_id: int, + raes_plan: RaesPlan, + runtime: RaesGceApplyRuntime | None, + mutation_started: bool, + options: RaesGceApplyOptions, +) -> None: + """Reconstructively clean up after a provider mutation, or release preflight state.""" + if mutation_started and runtime is not None: + logger.exception("RAES GCE range-cell apply failed; attempting cleanup request_id=%s", request_uuid) + _cleanup_failed_apply(request_uuid, range_id, raes_plan, runtime) + else: + logger.exception("RAES GCE range-cell apply failed before provider mutation request_id=%s", request_uuid) + if options.on_pre_mutation_failure is not None: + options.on_pre_mutation_failure() + + def apply_raes_range_cell( request_uuid: str, range_id: int, @@ -380,8 +423,7 @@ def apply_raes_range_cell( resolve_image, resolved_options, resolved_config, - delivery_bindings, - access_bindings, + _ApplyBindings(delivery_bindings, access_bindings), ) runtime = _apply_runtime(resolved_options, config=resolved_config) _preflight_existing_hosts(plan, runtime.clients) @@ -399,19 +441,10 @@ def apply_raes_range_cell( # A conflicting VM is not ours to delete, even if a race placed it # after the read-only preflight and some network resources were made. logger.exception("RAES GCE range-cell apply found a conflicting deterministic VM request_id=%s", request_uuid) - if mutation_started and runtime is not None and plan is not None: - _cleanup_created_resources(plan, raes_plan, runtime, created) - if not mutation_started and resolved_options.on_pre_mutation_failure is not None: - resolved_options.on_pre_mutation_failure() + _cleanup_conflicting_apply(plan, raes_plan, runtime, created, mutation_started, resolved_options) raise except Exception: - if mutation_started and runtime is not None: - logger.exception("RAES GCE range-cell apply failed; attempting cleanup request_id=%s", request_uuid) - _cleanup_failed_apply(request_uuid, range_id, raes_plan, runtime) - else: - logger.exception("RAES GCE range-cell apply failed before provider mutation request_id=%s", request_uuid) - if resolved_options.on_pre_mutation_failure is not None: - resolved_options.on_pre_mutation_failure() + _cleanup_failed_apply_attempt(request_uuid, range_id, raes_plan, runtime, mutation_started, resolved_options) raise return { "subnets": subnet_outputs(plan), diff --git a/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.tsx b/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.tsx index 3661adf0a..d006df72d 100644 --- a/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.tsx +++ b/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.tsx @@ -100,7 +100,7 @@ function validImageProfile(profile: AdapterTargetImageProfile): boolean { && Boolean(profile.domain_dns_name && profile.domain_netbios_name); } return profile.bootstrap_capability === PRECONFIGURED_HOST_CAPABILITY - && (profile.image_kind === "machine-image" + && (profile.image_kind === MACHINE_IMAGE_KIND ? /^projects\/[a-z0-9][a-z0-9.:-]*\/global\/machineImages\/[a-z][-a-z0-9]*$/.test(profile.image_ref) : /^projects\/[a-z0-9][a-z0-9.:-]*\/global\/images\/[a-z][-a-z0-9]*$/.test(profile.image_ref)) && Boolean(profile.management_ssh_username && profile.participant_container_name && profile.participant_username) From 220f40bdf6595e0fa6ef62375ffa8bac11662a90 Mon Sep 17 00:00:00 2001 From: Brad Edwards Date: Thu, 24 Sep 2026 06:33:20 +0000 Subject: [PATCH 5/6] style(gcp): document apply helpers and reuse readiness contract --- shifter/engine/provisioner/raes_gcp_apply.py | 4 ++++ .../src/features/administer/AdapterPackBindings.tsx | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/shifter/engine/provisioner/raes_gcp_apply.py b/shifter/engine/provisioner/raes_gcp_apply.py index 80f7e78c9..ef927b1ea 100644 --- a/shifter/engine/provisioner/raes_gcp_apply.py +++ b/shifter/engine/provisioner/raes_gcp_apply.py @@ -81,11 +81,15 @@ @dataclass(frozen=True, slots=True) class _ApplyBindings: + """Optional content and participant access projections admitted for one apply.""" + delivery: list[dict[str, Any]] | None access: list[dict[str, Any]] | None def _record_created(created: list[tuple[str, str]] | None, kind: str, name: str, was_created: bool) -> None: + """Track only resources inserted by the current apply attempt.""" + if was_created and created is not None: created.append((kind, name)) diff --git a/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.tsx b/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.tsx index d006df72d..02e8ba671 100644 --- a/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.tsx +++ b/shifter/shifter_platform/frontend/src/features/administer/AdapterPackBindings.tsx @@ -23,6 +23,7 @@ import { PackUploadForm } from "./PackUploadForm"; const MACHINE_IMAGE_KIND = "machine-image"; const BOOT_IMAGE_KIND = "image"; const PRECONFIGURED_HOST_CAPABILITY = "preconfigured-machine-host"; +const PARTICIPANT_READINESS_CONTRACT = "participant-readiness/v1"; function bindingStatus(pack: AdapterPack): string { if (!pack.binding) return "No adapter assigned"; @@ -104,7 +105,7 @@ function validImageProfile(profile: AdapterTargetImageProfile): boolean { ? /^projects\/[a-z0-9][a-z0-9.:-]*\/global\/machineImages\/[a-z][-a-z0-9]*$/.test(profile.image_ref) : /^projects\/[a-z0-9][a-z0-9.:-]*\/global\/images\/[a-z][-a-z0-9]*$/.test(profile.image_ref)) && Boolean(profile.management_ssh_username && profile.participant_container_name && profile.participant_username) - && profile.participant_readiness_contract === "participant-readiness/v1" + && profile.participant_readiness_contract === PARTICIPANT_READINESS_CONTRACT && /^[0-9a-f]{64}$/.test(profile.participant_readiness_manifest_sha256); } @@ -214,7 +215,7 @@ function TargetImageProfile({ name, value, onChange }: Readonly<{ onChange={(event) => { const machine = event.target.value === MACHINE_IMAGE_KIND; update({ image_kind: machine ? MACHINE_IMAGE_KIND : BOOT_IMAGE_KIND, bootstrap_capability: machine ? PRECONFIGURED_HOST_CAPABILITY : "standard", - participant_readiness_contract: machine ? "participant-readiness/v1" : "", + participant_readiness_contract: machine ? PARTICIPANT_READINESS_CONTRACT : "", participant_container_name: "", participant_username: "", participant_readiness_manifest_sha256: "", domain_dns_name: "", domain_netbios_name: "", }); }}> @@ -245,7 +246,7 @@ function TargetImageProfile({ name, value, onChange }: Readonly<{