diff --git a/docs/adr/index.yaml b/docs/adr/index.yaml index 72251aa0a..b99e4a213 100644 --- a/docs/adr/index.yaml +++ b/docs/adr/index.yaml @@ -1091,7 +1091,7 @@ }, { "id": "ADR-017-R5", - "description": "A workspace egress selector is resolved once at CMS launch admission while holding the workspace mutation mutex. The compatibility value inherits the normalized deployment baseline and `none` selects ADR-026 zero egress. The resulting closed mode is persisted on Engine Range, replay-verified, and transported in the exact operation-generation input beside (never inside) scenario/RAES contracts. Workspace identity and mutable policy documents do not cross into Engine provider adapters, task argv/environment, labels, events, or guest metadata.", + "description": "A workspace egress selector is resolved at CMS launch admission while holding the policy workspace mutation mutex. Ordinarily the policy workspace is the range's authorized ownership workspace. For a server-derived CTF participant or managed-spare launch, the immutable event workspace may instead supply policy while the range remains bound to its individual owner's workspace; event ownership and current event-workspace authority must be checked, and public callers may not supply this override. The compatibility value inherits the normalized deployment baseline and `none` selects ADR-026 zero egress. The resulting closed mode is persisted on Engine Range, replay-verified, and transported in the exact operation-generation input beside (never inside) scenario/RAES contracts. Workspace identity and mutable policy documents do not cross into Engine provider adapters, task argv/environment, labels, events, or guest metadata.", "checks": [] } ], @@ -1463,7 +1463,7 @@ }, { "id": "ADR-026-R6", - "description": "GCP zero egress must be realized behind the existing GCE range-cell provider seam with native Cloud NAT and VPC firewall resources. The stable `ALL_SUBNETWORKS_ALL_IP_RANGES` NAT posture is incompatible with a per-range `none` subnet and must be replaced through a coordinated migration that preserves NAT for existing/default ranges before it is removed. A firewall-only implementation, concurrent per-range patching of one Terraform-owned NAT object, or an unvalidated switch to the currently unreachable `vpc-per-range` mode is prohibited.", + "description": "GCP zero egress must be realized behind the existing GCE range-cell provider seam with native Cloud NAT and VPC firewall resources. A shared-VPC deployment may replace range-owned routers with region-scoped shared Router/NAT capacity only when one owner serializes explicit subnet enrollment and removal, reconciles provider drift, bounds each gateway by provider subnet/port/address limits, and never enrolls a `none` subnet. The Terraform migration-bridge NAT and provisioner-owned NAT must not manage the same subnet or mutate the same Router. The stable `ALL_SUBNETWORKS_ALL_IP_RANGES` NAT posture is incompatible with a per-range `none` subnet; migration must preserve existing/default-range NAT until each subnet is safely transferred. A firewall-only implementation, concurrent per-range patching of a Terraform-owned NAT object, or an unvalidated switch to `vpc-per-range` mode is prohibited.", "checks": [] } ], @@ -2810,7 +2810,7 @@ }, { "id": "ADR-046-R10", - "description": "Workspace quota and workspace range-policy evaluation compose at one CMS pre-reservation launch-admission seam parameterized by authorized workspace, individual owner, server-derived range source, trusted instantiation purpose, and stable server-minted request correlation. Enforcing concurrent-range quota uses a workspaces-owned durable idempotent reservation identified by workspace, resource, and correlation under the workspace mutation mutex rather than a count-then-create race, remains distinct from the per-user/source active-range constraint and Engine's event/provider-capacity ledger, and releases only on terminal FAILED/DESTROYED convergence (not DESTROYING or CMS soft delete). Member-seat quota composes once in the common locked membership insert used by direct add and invitation acceptance; membership rows remain canonical usage and pending invitations are not seats. Missing policy preserves unlimited compatibility; soft/advisory overage warns and records while hard/enforcing overage records then blocks without rolling its decision evidence back. Quota reads reuse READ_WORKSPACE authority; policy authoring is a service-backed, strict-audited superuser operation reached through the Django-admin escape hatch, never workspace-role authority or a raw model save. Quota decisions pin bounded policy/usage/reason facts as append-only workspaces-domain evidence, with applied-limit audit events emitted transactionally through shared.audit. The workspace egress selector extends the canonical installation RangeEgressMode vocabulary, resolves under the workspace mutation mutex, pins one closed effective decision on the Engine range, and is replay-verified; workspace identity, membership, role, quota/policy documents, and decisions never enter scenario/RAES contracts, provisioner argv/environment, events, provider labels, or guest metadata.", + "description": "Workspace quota and workspace range-policy evaluation compose at one CMS pre-reservation launch-admission seam parameterized by authorized ownership workspace, individual owner, server-derived range source, trusted instantiation purpose, and stable server-minted request correlation. Enforcing concurrent-range quota uses a workspaces-owned durable idempotent reservation identified by ownership workspace, resource, and correlation under the workspace mutation mutex rather than a count-then-create race, remains distinct from the per-user/source active-range constraint and Engine's event/provider-capacity ledger, and releases only on terminal FAILED/DESTROYED convergence (not DESTROYING or CMS soft delete). Member-seat quota composes once in the common locked membership insert used by direct add and invitation acceptance; membership rows remain canonical usage and pending invitations are not seats. Missing policy preserves unlimited compatibility; soft/advisory overage warns and records while hard/enforcing overage records then blocks without rolling its decision evidence back. Quota reads reuse READ_WORKSPACE authority; policy authoring is a service-backed, strict-audited superuser operation reached through the Django-admin escape hatch, never workspace-role authority or a raw model save. Quota decisions pin bounded policy/usage/reason facts as append-only workspaces-domain evidence, with applied-limit audit events emitted transactionally through shared.audit. The workspace egress selector extends the canonical installation RangeEgressMode vocabulary, resolves under its policy workspace mutation mutex, pins one closed effective decision on the Engine range, and is replay-verified. A trusted CTF participant/spare launch may use its immutable event workspace for that selector while retaining individual ownership workspace for scope and quota; two-workspace locking must be ordered and current authority rechecked. Workspace identity, membership, role, quota/policy documents, and decisions never enter scenario/RAES contracts, provisioner argv/environment, events, provider labels, or guest metadata.", "checks": [] }, { @@ -3676,7 +3676,7 @@ "title": "GCP live-fire range cells use provider-enforced isolation and participant-bounded identity", "status": "accepted", "scope": "gcp_range_plane", - "decision": "A compromised GCE live-fire guest, including root on a Docker host, is confined by provider-enforced per-range subnet/tag firewall policy, range-owned routing and NAT, a complete deny inventory of the management and range networks, and participant-bounded IAM. Participant-controlled VMs have no external NIC and default to no attached Google service account; a service account may be attached only for an explicitly approved capability that cannot use the existing provisioner bootstrap boundary, and its token is treated as participant-visible, range-bounded, and without Compute, IAM, project Secret Manager, shared Storage, or control-plane authority. Guest or container metadata firewalling is defense in depth: because a root guest can reach the GCE metadata endpoint, containment is proven by absence of a management or cross-range credential and by root-context effective-permission tests, with any retained narrow capability explicitly accepted. Range egress is range-owned: each non-zero-egress range gets its own Cloud NAT and a zero-egress range gets none; any shared NAT is a bounded migration bridge that enrolls only explicitly listed pre-migration subnets and never every subnet. Sanctioned egress lanes target the public-internet complement and operator allow-CIDRs that are validated to exclude the denied-network inventory, so a guest cannot reach management, peer-range, private-service, or metadata destinations through them. The canonical provider-neutral range-cell, egress, instantiation-policy, lifecycle, and escape-report contracts remain authoritative; GCP firewall/NAT/IAM are realizations below them. GDC VM Runtime, scenario Pods, L2 Networks, namespaces, and NetworkAttachmentDefinitions remain non-user validation resources and are not hardened or reopened as participant containment.", + "decision": "A compromised GCE live-fire guest, including root on a Docker host, is confined by provider-enforced per-range subnet/tag firewall policy, explicitly scoped NAT enrollment, a complete deny inventory of the management and range networks, and participant-bounded IAM. Participant-controlled VMs have no external NIC and default to no attached Google service account; a service account may be attached only for an explicitly approved capability that cannot use the existing provisioner bootstrap boundary, and its token is treated as participant-visible, range-bounded, and without Compute, IAM, project Secret Manager, shared Storage, or control-plane authority. Guest or container metadata firewalling is defense in depth: because a root guest can reach the GCE metadata endpoint, containment is proven by absence of a management or cross-range credential and by root-context effective-permission tests, with any retained narrow capability explicitly accepted. Each non-zero-egress range has its own explicit subnet membership on either its range-owned NAT or serialized provisioner-owned regional shared NAT; a zero-egress range has no NAT membership. Terraform's independently owned shared NAT remains only a bounded migration bridge for explicitly listed pre-migration subnets, never every subnet. Sanctioned egress lanes target the public-internet complement and operator allow-CIDRs that are validated to exclude the denied-network inventory, so a guest cannot reach management, peer-range, private-service, or metadata destinations through them. The canonical provider-neutral range-cell, egress, instantiation-policy, lifecycle, and escape-report contracts remain authoritative; GCP firewall/NAT/IAM are realizations below them. GDC VM Runtime, scenario Pods, L2 Networks, namespaces, and NetworkAttachmentDefinitions remain non-user validation resources and are not hardened or reopened as participant containment.", "rules": [ { "id": "ADR-056-R1", @@ -3685,7 +3685,7 @@ }, { "id": "ADR-056-R2", - "description": "Range egress is range-owned. Each non-zero-egress range gets its own regional Cloud NAT and a zero-egress (`none`) range gets none; any shared range NAT is a migration bridge that enrolls only the explicitly listed pre-migration subnets for its own region via LIST_OF_SUBNETWORKS and never ALL_SUBNETWORKS (which would NAT a zero-egress range, since a firewall deny does not remove NAT enrollment). The effective ADR-017 egress mode carried by shared.range_cells is the single policy vocabulary: mode `none` creates no range or shared NAT enrollment. Sanctioned public-web egress targets the public-internet complement of the denied-network inventory, and an operator egress allow-CIDR is rejected when it overlaps that inventory, so neither lane can name a management, peer-range, private-service, metadata, or special-use destination by rule precedence or accident. DNS transport is tested separately and may not be inferred from name-resolution failure.", + "description": "Range egress policy and subnet enrollment are range-scoped. A non-zero-egress range in a dedicated VPC gets its own regional Cloud NAT; in a shared VPC, its explicitly listed subnets may be enrolled on serialized, provisioner-owned regional shared NAT. A zero-egress (`none`) range gets no NAT membership. Terraform's separate shared NAT is only a migration bridge for explicitly listed pre-migration subnets; the provisioner may replay an existing bridge-attached range without double enrollment but must refuse subnet deletion until the bridge is drained by its Terraform owner. Neither shared NAT may use ALL_SUBNETWORKS (which would NAT a zero-egress range, since a firewall deny does not remove NAT enrollment). The effective ADR-017 egress mode carried by shared.range_cells is the single policy vocabulary. Sanctioned public-web egress targets the public-internet complement of the denied-network inventory, and an operator egress allow-CIDR is rejected when it overlaps that inventory, so neither lane can name a management, peer-range, private-service, metadata, or special-use destination by rule precedence or accident. DNS transport is tested separately and may not be inferred from name-resolution failure.", "checks": [] }, { diff --git a/docs/architecture/ctf-event-egress-preflight-2372.md b/docs/architecture/ctf-event-egress-preflight-2372.md new file mode 100644 index 000000000..424e23988 --- /dev/null +++ b/docs/architecture/ctf-event-egress-preflight-2372.md @@ -0,0 +1,36 @@ +# CTF event egress preflight (#2372) + +Status: pre-implementation guidance. The issue is the contract; no Ground Control requirement is attached. + +## Decisions and boundaries + +- Keep three identities distinct: `CTFEvent.workspace_id` is the immutable event and policy source; `RangeInstance.user`/Engine `Range.user` is the participant or managed spare owner; CMS Request, RangeInstance, and Engine Range `workspace_id` remain the owner's authorized workspace binding. A CTF event workspace must not become shared participant range ownership or grant its members terminal, VPN, range-list, or destroy access. Participant personal policy must not silently override the event's policy. A spare's new personal workspace must not supply its launch policy. +- CTF derives event ID, event owner, participant or spare identity, and event workspace server-side. Its event-ownership and participation checks remain CTF-owned. The trusted `ctf.bridges` to `cms.services` launch contract may carry a narrowly scoped CTF policy-workspace selector; the public range API must not accept one. CMS validates the CTF source and current event-workspace authority through `workspaces.services`, keeps its existing user/workspace launch and content gates, and resolves the policy under the workspace mutation lock when reserving the range. If locking both ownership and policy workspaces, use a stable lock order. Failure must be opaque and fail closed, never fall back to a personal policy. `content_authorizer` proves package visibility only; it is not policy or launch authorization. +- Reuse `workspaces.services.set_workspace_egress_policy` and its owner/admin authorization and strict old/new audit. Event workspace policy changes affect subsequent reservations, not active ranges. Pin only the closed effective `RangeEgressMode` on the Engine range and replay-verify it; record the source workspace and effective mode in bounded CMS/CTF audit evidence, without sending workspace identity or mutable policy into provisioner jobs, provider labels, guest metadata, or RAES scenario data. A no-op policy change emits no audit event. +- Spare transfer is an ownership transaction, not a fresh network-policy decision. `cms.services.reassign_range_owner` updates CMS and Engine ownership and explicitly rehomes workspace scope. The already pinned egress mode remains unchanged; CTF must refuse or replace an incompatible spare if the event policy changed after it was provisioned. Recovery rebuilds and scheduled/manual participant launches use the same policy resolution. Preserve the participant assignment lock, per-user/source active-range constraint, event capacity draw, workspace quota reservation, and retry classification. +- Shared-VPC egress needs region-scoped shared capacity with **one mutation owner**. The GCE range-cell provider must enroll only non-`none` participant subnets into explicitly listed Cloud NAT subnet ranges, serialize enrollment/removal across concurrent jobs, reconcile actual Router/NAT state before writes, and make replay/destroy idempotent. Separate provisioner-owned shared NAT from Terraform's existing migration-bridge NAT; Terraform apply and range jobs must never write the same Router or enroll the same subnet twice. Remove a subnet from NAT before deleting it. Keep per-range firewall rules and network tags as the isolation gate. Never use `ALL_SUBNETWORKS_ALL_IP_RANGES`, external guest IPs, or a firewall-only approximation of zero egress. +- Shard shared NAT by region and gateway capacity rather than by range router. Google's documented current limits are five Cloud Routers per VPC network per region, 50 NAT gateways per Router, and 50 explicitly listed subnet ranges per gateway; these are upper bounds, not a promised participant count. Expected capacity is the minimum of available gateway/subnet slots, range subnet and IP space, NAT IP/port budget, instance and address quotas, and the event capacity assessment. Admit before resource creation when any bound is exhausted; report a bounded reason code and operator-visible headroom. Size by the actual subnets and VMs per range, including spares, not by participant count alone. See [Cloud NAT limits](https://docs.cloud.google.com/nat/quota) and [NAT port allocation](https://docs.cloud.google.com/nat/docs/ports-and-addresses). Regional outage and NAT exhaustion remain shared failure domains. +- Participant web research uses the existing `GCERangeImageProfile.allow_public_web_egress` boolean and `gcp_range_cell_firewall.build_firewall_plan` public IPv4 complement on TCP 80/443. It remains off by default and must be ineffective under `none` or `deny-all`; preserve the denied-network inventory, private Google API lane, management exclusion, and per-range default deny. Do not add an event-wide `allow internet` flag or a scenario/pack authority for this profile-level capability. + +## Canonical paths to reuse + +| Concern | Existing path and constraint | +| --- | --- | +| Event scope and auth | `ctf.services.event._workspace`, `CTFEvent.workspace_id`, `ctf.services.authorization`, organizer and participant API gates. Event membership is distinct from workspace membership. | +| Launch and ownership | `ctf.services.range.{provision,spares,recovery}`, `ctf.bridges.CTFRangeLaunchOptions`, `cms.services._raes_range_create`, `_range_workspace`, `_range_launch_common`, `_range_reassign`; use the CMS facade, request correlation, locked reservation, quota, and Engine dispatch. | +| Egress vocabulary and config | `installation.range_egress.RangeEgressPolicy`/`RangeEgressMode`, `workspaces.services._egress`, `cms.services._range_workspace.resolve_effective_egress_mode_locked`, `shared.range_instantiation_policy`; no parallel enum, policy JSON, or CIDR validator. | +| Provider and capacity | `engine/provisioner/gcp_range_cell_{plan,cells,resources,destroy,firewall}`, `components/network` coordination pattern, `ctf.services.range.capacity`, Engine capacity inventory/reservation, `platform/terraform/gcp/modules/range/vpc` migration bridge. Do not use provider quota as a substitute for the event budget. | +| Error, audit, observability | `CTFRangeError`, `CMSError`/`WorkspaceLaunchDenied`, `shared.api.errors`, `shared.audit`, `ctf.services.audit`, `shared.log_sanitize`; keep public denials non-enumerating and logs limited to sanitized IDs, mode, bounded reason codes and aggregate capacity. | +| Verification | Extend DB-backed CTF/CMS/workspace tests and GCE plan/resource/destroy tests. Exercise distinct participant personal workspaces, managed spares, a differently configured event workspace, policy changes between launches, recovery handover, concurrent enrollment, replay, rollback, partial destroy, and `none` negative evidence. Test guest reachability as well as planned rules. | + +## Whole-repository gates and gotchas + +- **Auth and validation:** CTF API serializers, session/API-token scopes, event ownership, participant eligibility, `workspaces.services` operation checks, CMS user/scenario/backend/source gates, locked workspace admission, shared egress enum, Engine operation-generation validation, GCE profile JSON parser, and firewall denied-network validation must all agree. A service caller bypassing an HTTP serializer must still be rejected. Do not treat organizer ownership, CTF role, `content_authorizer`, or a bare internal workspace ID as a workspace-policy grant. +- **Config, secrets, and runtime:** `settings.range_egress` is the deployment baseline; `GCP_RANGE_IMAGE_KEY_PROFILES_JSON` is the closed profile-level web opt-in. If runtime bindings change, update `config/_gce*`, Helm values/schema, GCP Kustomize, both provisioner-job admission policies, and Terraform root/module validation together. Keep credentials, invite tokens, flags, provider payloads, URLs, env dumps, and policy source identity out of argv, job environment additions, logs, error envelopes, and guest metadata. Do not add a second env schema for event policy. +- **Persistence and exceptions:** Preserve CMS Request/RangeInstance and Engine Range workspace projections and IDs; CTF participant/spare IDs are not range IDs. Policy and ownership audit writes must be durable with the corresponding mutation. Reuse existing exception classes and authored, opaque API envelopes; raw provider errors stay operator-side. +- **Network and workflow:** GCP regional placement and provider quotas, `platform/terraform/gcp/modules/range/vpc` state ownership, IAM for Router/NAT mutation, range-cell create/destroy compensation, scheduler heartbeat and retry, event capacity admission, and cloud quota/port metrics are in scope. Avoid per-range Routers, global-all-subnet NAT, uncoordinated Router patching, NAT enrollment of `none`, or silent reuse of a stale NAT/config body. A router name existing is not proof its subnet list is correct. +- **Checks:** Changes to architecture/guardrails or `shifter/shifter_platform` require `python3 scripts/adr_guard/adr_guard.py --all --level ci`; relevant implementation also needs import-linter, GCP Terraform validation/tests, and the subsystem tests above. ADR-017-R5 and ADR-026-R6 govern this design. + +## Non-goals + +No product code, migration, or cloud resource is changed by this preflight. The implementation need not redesign generic interactive workspace selection, workspace RBAC, CTF membership, AWS egress, RAES contracts, model access, private packs, or the range access protocols. The event policy source is a narrow CTF launch fact, not a new tenant-wide default. diff --git a/docs/dev/gcp-range-cell-deploy.md b/docs/dev/gcp-range-cell-deploy.md index ee7465870..b86dc1285 100644 --- a/docs/dev/gcp-range-cell-deploy.md +++ b/docs/dev/gcp-range-cell-deploy.md @@ -178,6 +178,40 @@ control-plane `GCP_PROJECT_ID` is a deploy-overlay placeholder. retained as a selectable mode for a future peering/IAP implementation; do not use it for live deployments yet. +### Shared-VPC NAT capacity and event egress + +Non-zero-egress range subnets are enrolled explicitly in a provisioner-owned +regional Cloud Router/NAT. The provisioner serializes Router changes and removes +each subnet from NAT before deleting it. This Router is separate from the +Terraform-owned migration-bridge NAT; do not configure both to cover the same +subnet. A `none` range is never enrolled, and a conflicting all-subnet NAT +causes admission to fail rather than relying on a firewall-only deny. +An existing range whose already-created subnets are still explicitly listed on +the Terraform bridge can replay without duplicate NAT enrollment. Before +destroying such a range, remove those subnet links from the Terraform bridge +through that environment's deploy workflow; destroy refuses while the bridge +still owns them. New subnets are never admitted onto the bridge. + +Cloud NAT currently allows [50 explicit subnet ranges per gateway and 50 +gateways per Router](https://docs.cloud.google.com/nat/quota); a VPC can have up +to five Cloud Routers per region. The provisioner uses one regional Router and +shards explicit subnets across its gateways, so plan against *subnets per +range*, including recovery spares, not only participant count. Its 2,500 +explicit-subnet theoretical ceiling is not a guaranteed event size: NAT +[port/IP allocation](https://docs.cloud.google.com/nat/docs/ports-and-addresses), +subnet/IP space, VMs, addresses, image launch rates, and the event capacity +assessment can bind first. Review provider quota and NAT utilization before a +large event; shared NAT is a regional failure domain. A capacity or conflicting +scope refusal must be corrected at the source/configuration boundary, not by +adding an all-subnet gateway or a one-off range firewall rule. + +For CTF participant and managed-spare launches, the event workspace supplies +the egress policy while each range remains owned by its participant or spare +account and bound to that account's authorized workspace. Workspace policy +changes are audited and affect subsequent range reservations only. Already +launched ranges retain their pinned mode; a recovery spare whose mode no longer +matches the event is refused before the old participant range is torn down. + ## Legacy RangeSpec image mapping The scenario-owned legacy `RangeSpec` adapter resolves current instances to one diff --git a/docs/features/ctf-organizer-guide.md b/docs/features/ctf-organizer-guide.md index 9fcf4e570..b106a85fe 100644 --- a/docs/features/ctf-organizer-guide.md +++ b/docs/features/ctf-organizer-guide.md @@ -33,6 +33,15 @@ From **CTF Admin → Events → Create**, set the core parameters: | Auto cleanup / cleanup delay | Whether ranges are torn down after the event | | Publish public registration page | Opt-in public event details and request form; off by default | +The event's workspace supplies the egress policy for new participant and +recovery-spare ranges. Each range still belongs to its participant or managed +spare account; selecting a shared event workspace does not give other workspace +members access to those ranges. Set the workspace policy before provisioning: +changes are audited and apply to later launches, not ranges already running. +If the policy changes after a spare is prepared, an incompatible spare cannot +replace a participant range. For large GCP events, include both participants +and spares in the [shared-VPC NAT capacity check](../dev/gcp-range-cell-deploy.md#shared-vpc-nat-capacity-and-event-egress). + ### Optional public registration Enable **Publish public registration page** only when you want an unauthenticated diff --git a/docs/requirements/PLAT-238/requirement.md b/docs/requirements/PLAT-238/requirement.md index da6d3dcb8..bc2d66690 100644 --- a/docs/requirements/PLAT-238/requirement.md +++ b/docs/requirements/PLAT-238/requirement.md @@ -21,3 +21,5 @@ Program #1321 specifies zero-egress (#1171) be delivered as a workspace-level po ## Traceability - IMPLEMENTS → GITHUB_ISSUE `1945` +- IMPLEMENTS → CODE `shifter/engine/provisioner/gcp_range_cell_shared_nat.py` (Explicit shared-VPC NAT enrollment preserves zero-egress ranges and bounded GCP capacity) +- TESTS → TEST `shifter/engine/provisioner/tests/test_gcp_shared_nat.py` (Shared-NAT capacity, zero-egress, and migration replay coverage) diff --git a/docs/technical/platform_infrastructure/gcp-infrastructure.md b/docs/technical/platform_infrastructure/gcp-infrastructure.md index 4ffc67a08..fd921f8ce 100644 --- a/docs/technical/platform_infrastructure/gcp-infrastructure.md +++ b/docs/technical/platform_infrastructure/gcp-infrastructure.md @@ -95,7 +95,7 @@ Dual-network design, same pattern as AWS (see [Networking](networking)). Networks are peered bidirectionally for platform-to-range connectivity. -Live-fire ranges use GCE range cells (a per-range subnet with tag firewall isolation and range-owned NAT) as the approved containment boundary per ADR-030 and ADR-039, with the containment model in ADR-056. GDC custom L2 networks (VXLAN-based) for per-range guest isolation are a dev and operator-validation path only, not a live-fire containment boundary. +Live-fire ranges use GCE range cells (per-range subnets with tag firewall isolation and explicit NAT enrollment) as the approved containment boundary per ADR-030 and ADR-039, with the containment model in ADR-056. Shared-VPC ranges use a provisioner-owned regional Router/NAT with only admitted subnets listed; VPC-per-range cells retain a range-owned Router/NAT. See the [range-cell deploy runbook](../../dev/gcp-range-cell-deploy.md#shared-vpc-nat-capacity-and-event-egress) for capacity and lifecycle bounds. GDC custom L2 networks (VXLAN-based) for per-range guest isolation are a dev and operator-validation path only, not a live-fire containment boundary. ## Deployment Path diff --git a/shifter/engine/provisioner/gcp_range_cell_clients.py b/shifter/engine/provisioner/gcp_range_cell_clients.py index 9dcf9d7b2..b9785a7af 100644 --- a/shifter/engine/provisioner/gcp_range_cell_clients.py +++ b/shifter/engine/provisioner/gcp_range_cell_clients.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Iterable from dataclasses import dataclass from typing import Protocol @@ -31,6 +32,16 @@ def patch(self, **kwargs: object) -> object: """Converge one existing firewall rule to a new body and return an operation.""" +class RoutersCollectionClient(ComputeCollectionClient, Protocol): + """Router operations including explicit NAT membership reconciliation.""" + + def patch(self, **kwargs: object) -> object: + """Patch NAT gateway membership after a serialized fresh read.""" + + def list(self, **kwargs: object) -> Iterable[object]: + """List regional routers to validate quota and conflicting NAT scopes.""" + + class ComputeInstancesClient(ComputeCollectionClient, Protocol): """Compute instance operations additionally used by range lifecycle.""" @@ -65,7 +76,7 @@ class GCEClients: subnetworks: ComputeCollectionClient firewalls: FirewallsCollectionClient addresses: ComputeCollectionClient - routers: ComputeCollectionClient + routers: RoutersCollectionClient instances: ComputeInstancesClient global_operations: OperationWaitClient region_operations: OperationWaitClient diff --git a/shifter/engine/provisioner/gcp_range_cell_destroy.py b/shifter/engine/provisioner/gcp_range_cell_destroy.py index 1e31f3b9b..03cac4216 100644 --- a/shifter/engine/provisioner/gcp_range_cell_destroy.py +++ b/shifter/engine/provisioner/gcp_range_cell_destroy.py @@ -14,6 +14,7 @@ ) from gcp_range_cell_ops import _delete_resource, _get_or_none, _wait_for_operation from gcp_range_cell_plan import render_range_cell_plan +from gcp_range_cell_shared_nat import remove_shared_nat from gcp_range_cell_types import RangeCellPlan, ResourceDict from provisioner_db import get_range_data_by_request_id from range_placement import resolve_placement_from_range_data @@ -108,6 +109,7 @@ def _destroy_network_resources(plan: RangeCellPlan, clients: GCEClients) -> None """Delete the range-owned router/NAT, firewalls, subnets, and (when range-owned) the VPC.""" # The range-owned Cloud Router (carrying the Cloud NAT) references this range's # subnets, so it is torn down before them (PLAT-238). Absent for a `none` range. + remove_shared_nat(plan, clients) router_nat = plan.get("router_nat") if router_nat is not None: _delete_resource( diff --git a/shifter/engine/provisioner/gcp_range_cell_naming.py b/shifter/engine/provisioner/gcp_range_cell_naming.py index 85be1ceac..09731a7b6 100644 --- a/shifter/engine/provisioner/gcp_range_cell_naming.py +++ b/shifter/engine/provisioner/gcp_range_cell_naming.py @@ -51,6 +51,17 @@ def range_router_nat_plan(range_id: int, subnet_self_links: list[str]) -> dict[s } +def shared_router_nat_plan(network_name: str, subnet_self_links: list[str]) -> dict[str, object]: + """Name the provisioner-owned regional NAT independently of any range or Terraform bridge.""" + return { + "router_name": _short_resource_name("shifter", network_name, "nat-router"), + # Allocator appends -0 through -49; reserve three characters under GCE's + # 63-character router-NAT name limit. + "nat_name": _short_resource_name("shifter", network_name, "nat", max_length=60), + "subnetwork_self_links": list(subnet_self_links), + } + + def _network_self_link(project_id: str, network_name: str) -> str: """Return the relative self-link for a global Compute network.""" return f"projects/{project_id}/global/networks/{network_name}" diff --git a/shifter/engine/provisioner/gcp_range_cell_plan.py b/shifter/engine/provisioner/gcp_range_cell_plan.py index 303b31429..069ed59d9 100644 --- a/shifter/engine/provisioner/gcp_range_cell_plan.py +++ b/shifter/engine/provisioner/gcp_range_cell_plan.py @@ -18,6 +18,7 @@ _subnet_tag, _subnetwork_self_link, range_router_nat_plan, + shared_router_nat_plan, ) from gcp_range_cell_scenario import build_instance_plans, realize_range_spec from gcp_range_cell_types import ( @@ -331,8 +332,12 @@ def render_range_cell_plan( # a `none` (zero-egress) range omits it so its subnets carry no NAT path # (PLAT-238, ADR-026-R6), mirroring the RAES plan builder. if egress_policy.mode.strip().lower() != "none": - plan["router_nat"] = cast( - RouterNatPlan, - range_router_nat_plan(range_id, [subnet["self_link"] for subnet in subnet_plans]), - ) + if manage_network: + plan["router_nat"] = cast( + RouterNatPlan, range_router_nat_plan(range_id, [subnet["self_link"] for subnet in subnet_plans]) + ) + else: + plan["shared_nat"] = cast( + RouterNatPlan, shared_router_nat_plan(network_name, [subnet["self_link"] for subnet in subnet_plans]) + ) return plan diff --git a/shifter/engine/provisioner/gcp_range_cell_shared_nat.py b/shifter/engine/provisioner/gcp_range_cell_shared_nat.py new file mode 100644 index 000000000..f42aa2b71 --- /dev/null +++ b/shifter/engine/provisioner/gcp_range_cell_shared_nat.py @@ -0,0 +1,340 @@ +"""Serialized, explicitly scoped Cloud NAT enrollment for shared range VPCs.""" + +from __future__ import annotations + +import hashlib +from collections.abc import Iterable, Iterator, Mapping, Sequence +from contextlib import contextmanager + +from gcp_range_cell_clients import GCEClients +from gcp_range_cell_naming import range_router_nat_plan +from gcp_range_cell_ops import _get_or_none, _wait_for_operation +from gcp_range_cell_types import RangeCellPlan +from provisioner_db import get_db_connection + +_SUBNETS_PER_GATEWAY = 50 +_GATEWAYS_PER_ROUTER = 50 + + +def _canonical_subnet_link(link: str) -> str: + """Normalize provider full and planned relative self-links to one identity.""" + _prefix, separator, suffix = link.partition("projects/") + if not separator or "/regions/" not in suffix or "/subnetworks/" not in suffix: + raise RuntimeError("shared-nat-invalid-subnet-link") + return f"projects/{suffix}" + + +def allocate_explicit_subnets( + existing: Mapping[str, Sequence[str]], requested: Sequence[str], *, nat_name: str +) -> list[dict[str, object]]: + """Preserve existing assignments while packing new subnets into bounded gateways.""" + slots = {name: [_canonical_subnet_link(link) for link in links] for name, links in existing.items()} + seen = {link for links in slots.values() for link in links} + for raw_link in requested: + link = _canonical_subnet_link(raw_link) + if link in seen: + continue + destination = next((name for name, links in slots.items() if len(links) < _SUBNETS_PER_GATEWAY), None) + if destination is None: + if len(slots) >= _GATEWAYS_PER_ROUTER: + raise RuntimeError("shared-nat-capacity-exhausted") + destination = next( + f"{nat_name}-{index}" for index in range(_GATEWAYS_PER_ROUTER) if f"{nat_name}-{index}" not in slots + ) + slots[destination] = [] + slots[destination].append(link) + seen.add(link) + return [ + { + "name": name, + "nat_ip_allocate_option": "AUTO_ONLY", + "source_subnetwork_ip_ranges_to_nat": "LIST_OF_SUBNETWORKS", + "subnetworks": [{"name": link, "source_ip_ranges_to_nat": ["ALL_IP_RANGES"]} for link in links], + } + for name, links in slots.items() + if links + ] + + +@contextmanager +def _regional_nat_lock(plan: RangeCellPlan) -> Iterator[None]: + """Serialize all provider NAT mutations for one shared network and region.""" + scope = f"{plan['project_id']}|{plan['region']}|{plan['network']['self_link']}".encode() + key = int.from_bytes(hashlib.sha256(scope).digest()[:8], "big", signed=True) + with get_db_connection() as connection, connection.cursor() as cursor: + cursor.execute("SET lock_timeout = '30s'") + cursor.execute("SELECT pg_advisory_lock(%s)", (key,)) + try: + yield + finally: + cursor.execute("SELECT pg_advisory_unlock(%s)", (key,)) + + +def _field(value: object, name: str) -> object: + """Read a field from either a provider object or its test dictionary.""" + return value.get(name) if isinstance(value, dict) else getattr(value, name, None) + + +def _items(value: object) -> Iterable[object]: + """Treat absent and scalar provider fields as empty collections.""" + if isinstance(value, Iterable) and not isinstance(value, (str, bytes, Mapping)): + return value + return () + + +def _router(plan: RangeCellPlan, clients: GCEClients, name: str) -> object | None: + """Read one regional router, returning None for a provider 404.""" + return _get_or_none( + clients.routers.get, + clients.google_exceptions, + project=plan["project_id"], + region=plan["region"], + router=name, + ) + + +def _gateway_subnets(plan: RangeCellPlan, router: object) -> dict[str, list[str]]: + """Validate ownership and return explicit membership by gateway.""" + network = str(_field(router, "network") or "") + if not network.endswith(plan["network"]["self_link"]): + raise RuntimeError("shared-nat-network-mismatch") + expected_prefix = f"{plan['shared_nat']['nat_name']}-" + result: dict[str, list[str]] = {} + for gateway in _items(_field(router, "nats")): + name = str(_field(gateway, "name") or "") + mode = _field(gateway, "source_subnetwork_ip_ranges_to_nat") + mode_name = getattr(mode, "name", str(mode)) + if not name.startswith(expected_prefix) or mode_name != "LIST_OF_SUBNETWORKS": + raise RuntimeError("shared-nat-ownership-mismatch") + result[name] = [ + _canonical_subnet_link(str(_field(item, "name") or "")) for item in _items(_field(gateway, "subnetworks")) + ] + return result + + +def _bridge_router_names(plan: RangeCellPlan) -> set[str]: + """Terraform's pre-migration NAT has a deterministic name on the range VPC.""" + network_name = str(plan["network"].get("name") or plan["network"]["self_link"].rsplit("/", 1)[-1]) + return {f"{network_name}-nat", f"{network_name}-nat-{plan['region']}"} + + +def _bridge_covered_subnets(plan: RangeCellPlan, routers: Iterable[object]) -> set[str]: + """Collect only explicit manual NAT membership on Terraform's bridge.""" + covered: set[str] = set() + bridge_names = _bridge_router_names(plan) + for router in routers: + if str(_field(router, "name") or "") not in bridge_names: + continue + for gateway in _items(_field(router, "nats")): + allocation = _field(gateway, "nat_ip_allocate_option") + if getattr(allocation, "name", str(allocation)) != "MANUAL_ONLY": + continue + scope = _field(gateway, "source_subnetwork_ip_ranges_to_nat") + if getattr(scope, "name", str(scope)) != "LIST_OF_SUBNETWORKS": + continue + covered.update( + _canonical_subnet_link(str(_field(subnet, "name") or "")) + for subnet in _items(_field(gateway, "subnetworks")) + ) + return covered + + +def _bridge_subnets_exist(plan: RangeCellPlan, clients: GCEClients, desired: set[str]) -> bool: + """A bridge may replay only subnets that still belong to this network.""" + for link in desired: + subnet = _get_or_none( + clients.subnetworks.get, + clients.google_exceptions, + project=plan["project_id"], + region=plan["region"], + subnetwork=link.rsplit("/", 1)[-1], + ) + if subnet is None or not str(_field(subnet, "network") or "").endswith(plan["network"]["self_link"]): + return False + return True + + +def _bridge_replay(plan: RangeCellPlan, clients: GCEClients, routers: Iterable[object]) -> bool: + """Keep an existing range on Terraform's bridge; never enroll a new subnet there.""" + nat = plan.get("shared_nat") + if nat is None: + return False + desired = {_canonical_subnet_link(link) for link in nat["subnetwork_self_links"]} + return ( + bool(desired) + and desired.issubset(_bridge_covered_subnets(plan, routers)) + and _bridge_subnets_exist(plan, clients, desired) + ) + + +def _verify_membership(plan: RangeCellPlan, clients: GCEClients, *, present: bool) -> None: + """Fail closed when provider readback disagrees with the requested mutation.""" + nat = plan["shared_nat"] + router = _router(plan, clients, nat["router_name"]) + actual = set() + if router is not None: + actual = {link for links in _gateway_subnets(plan, router).values() for link in links} + requested = set(nat["subnetwork_self_links"]) + if (present and not requested.issubset(actual)) or (not present and requested.intersection(actual)): + raise RuntimeError("shared-nat-reconciliation-incomplete") + + +def _same_network_routers(plan: RangeCellPlan, clients: GCEClients) -> list[object]: + """List only routers attached to the range's regional network.""" + regional = clients.routers.list(project=plan["project_id"], region=plan["region"]) + return [ + router for router in regional if str(_field(router, "network") or "").endswith(plan["network"]["self_link"]) + ] + + +def _assert_gateway_scope( + gateway: object, + *, + desired: set[str], + owner_name: str, + allowed_names: set[str], + bridge_owner: bool, + zero_egress: bool, +) -> None: + """Reject unbounded NAT or conflicting ownership of requested subnets.""" + scope = _field(gateway, "source_subnetwork_ip_ranges_to_nat") + if getattr(scope, "name", str(scope)) != "LIST_OF_SUBNETWORKS": + raise RuntimeError("shared-nat-unbounded-or-unknown-scope") + listed = {str(_field(subnet, "name") or "") for subnet in _items(_field(gateway, "subnetworks"))} + overlap = any(any(link.endswith(want) for link in listed) for want in desired) + if overlap and owner_name not in allowed_names and not bridge_owner: + raise RuntimeError("shared-nat-foreign-overlap") + if overlap and zero_egress: + raise RuntimeError("shared-nat-zero-egress-conflict") + + +def _assert_existing_nat_scopes( + plan: RangeCellPlan, + same_network: Sequence[object], + *, + legacy_name: str, + bridge_replay: bool, +) -> None: + """Reject broad or foreign NAT coverage before allocating another range.""" + desired = {str(subnet["self_link"]) for subnet in plan["subnets"]} + own = plan.get("shared_nat") + allowed_names = {legacy_name} | ({own["router_name"]} if own is not None else set()) + bridge_names = _bridge_router_names(plan) + for router in same_network: + name = str(_field(router, "name") or "") + for gateway in _items(_field(router, "nats")): + _assert_gateway_scope( + gateway, + desired=desired, + owner_name=name, + allowed_names=allowed_names, + bridge_owner=bridge_replay and name in bridge_names, + zero_egress=own is None, + ) + + +def _assert_shared_capacity_locked(plan: RangeCellPlan, clients: GCEClients) -> None: + """Check regional quota and all existing NAT scopes under the mutation lock.""" + own = plan.get("shared_nat") + legacy_name = str(range_router_nat_plan(plan["range_id"], [])["router_name"]) + same_network = _same_network_routers(plan, clients) + bridge_replay = _bridge_replay(plan, clients, same_network) + _assert_existing_nat_scopes(plan, same_network, legacy_name=legacy_name, bridge_replay=bridge_replay) + names = {str(_field(router, "name") or "") for router in same_network} + if own is None or legacy_name in names or bridge_replay: + return + if own["router_name"] not in names and len(same_network) >= 5: + raise RuntimeError("shared-nat-router-capacity-exhausted") + current = _router(plan, clients, own["router_name"]) + enrolled = _gateway_subnets(plan, current) if current is not None else {} + allocate_explicit_subnets(enrolled, own["subnetwork_self_links"], nat_name=own["nat_name"]) + + +def assert_shared_nat_capacity(plan: RangeCellPlan, clients: GCEClients) -> None: + """Reject quota exhaustion or a conflicting NAT before any range resource is made.""" + if not plan["manage_network"]: + with _regional_nat_lock(plan): + _assert_shared_capacity_locked(plan, clients) + + +def _ensure_shared_nat_locked(plan: RangeCellPlan, clients: GCEClients) -> None: + """Mutate shared NAT only after the regional advisory lock is held.""" + nat = plan["shared_nat"] + regional = clients.routers.list(project=plan["project_id"], region=plan["region"]) + # Terraform retains the old subnet until the operator drains its bridge. + # Existing ranges retain independent NAT until teardown. + legacy_name = range_router_nat_plan(plan["range_id"], [])["router_name"] + if _bridge_replay(plan, clients, regional) or _router(plan, clients, str(legacy_name)) is not None: + return + current = _router(plan, clients, nat["router_name"]) + enrolled = _gateway_subnets(plan, current) if current is not None else {} + gateways = allocate_explicit_subnets(enrolled, nat["subnetwork_self_links"], nat_name=nat["nat_name"]) + if current is None: + operation = clients.routers.insert( + project=plan["project_id"], + region=plan["region"], + router_resource={ + "name": nat["router_name"], + "network": plan["network"]["self_link"], + "region": plan["region"], + "nats": gateways, + }, + ) + elif {link for links in enrolled.values() for link in links} >= set(nat["subnetwork_self_links"]): + return + else: + operation = clients.routers.patch( + project=plan["project_id"], + region=plan["region"], + router=nat["router_name"], + router_resource={"nats": gateways}, + ) + _wait_for_operation(plan, clients, operation, "region") + _verify_membership(plan, clients, present=True) + + +def ensure_shared_nat(plan: RangeCellPlan, clients: GCEClients) -> None: + """Enroll this range's subnets in a provisioner-owned, explicit regional NAT.""" + if plan.get("shared_nat") is not None: + with _regional_nat_lock(plan): + _ensure_shared_nat_locked(plan, clients) + + +def _remove_shared_nat_locked(plan: RangeCellPlan, clients: GCEClients) -> None: + """Remove the legacy router or shared membership under the regional lock.""" + nat = plan["shared_nat"] + regional = clients.routers.list(project=plan["project_id"], region=plan["region"]) + if _bridge_replay(plan, clients, regional): + raise RuntimeError("shared-nat-migration-bridge-attached") + legacy_name = str(range_router_nat_plan(plan["range_id"], [])["router_name"]) + if _router(plan, clients, legacy_name) is not None: + operation = clients.routers.delete(project=plan["project_id"], region=plan["region"], router=legacy_name) + _wait_for_operation(plan, clients, operation, "region") + return + current = _router(plan, clients, nat["router_name"]) + if current is None: + return + enrolled = _gateway_subnets(plan, current) + requested = set(nat["subnetwork_self_links"]) + remaining = {name: [link for link in links if link not in requested] for name, links in enrolled.items()} + if remaining == enrolled: + return + gateways = allocate_explicit_subnets(remaining, [], nat_name=nat["nat_name"]) + if gateways: + operation = clients.routers.patch( + project=plan["project_id"], + region=plan["region"], + router=nat["router_name"], + router_resource={"nats": gateways}, + ) + else: + operation = clients.routers.delete(project=plan["project_id"], region=plan["region"], router=nat["router_name"]) + _wait_for_operation(plan, clients, operation, "region") + _verify_membership(plan, clients, present=False) + + +def remove_shared_nat(plan: RangeCellPlan, clients: GCEClients) -> None: + """Remove this range's NAT entries before its subnets are deleted.""" + if plan.get("shared_nat") is not None: + with _regional_nat_lock(plan): + _remove_shared_nat_locked(plan, clients) diff --git a/shifter/engine/provisioner/gcp_range_cell_types.py b/shifter/engine/provisioner/gcp_range_cell_types.py index f3f0f2356..1b304c3e8 100644 --- a/shifter/engine/provisioner/gcp_range_cell_types.py +++ b/shifter/engine/provisioner/gcp_range_cell_types.py @@ -145,6 +145,7 @@ class RangeCellPlan(TypedDict): # A `none` (zero-egress) range omits it entirely, so its subnets carry no NAT # path at all -- a firewall deny alone is not that guarantee. router_nat: NotRequired[RouterNatPlan] + shared_nat: NotRequired[RouterNatPlan] class RouterNatPlan(TypedDict): diff --git a/shifter/engine/provisioner/gcp_range_cells.py b/shifter/engine/provisioner/gcp_range_cells.py index 5a914a6b7..9ac97aecb 100644 --- a/shifter/engine/provisioner/gcp_range_cells.py +++ b/shifter/engine/provisioner/gcp_range_cells.py @@ -33,6 +33,7 @@ router_nat_resource, subnetwork_resource, ) +from gcp_range_cell_shared_nat import assert_shared_nat_capacity, ensure_shared_nat from gcp_range_cell_types import ( FirewallPlan, InstancePlan, @@ -120,8 +121,13 @@ def _ensure_router_nat(plan: RangeCellPlan, clients: GCEClients) -> bool: element and therefore no NAT path. Idempotent: an existing router of the same name is left in place (the NAT config is deterministic from the plan). """ + shared_nat = plan.get("shared_nat") + if shared_nat is not None: + ensure_shared_nat(plan, clients) router_nat = plan.get("router_nat") - if router_nat is None: + # A shared-NAT range delegates egress to the shared router; a zero-egress + # range carries no router_nat. Neither owns a range-scoped router here. + if shared_nat is not None or router_nat is None: return False name = router_nat["router_name"] existing = _get_or_none( @@ -307,6 +313,7 @@ def _provision_range_resources( shared-vpc mode the pre-existing platform-peered VPC is reused and only the per-range subnets/firewalls/instances are created here. """ + assert_shared_nat_capacity(plan, clients) if plan["manage_network"]: _ensure_network(plan, clients) for subnet in plan["subnets"]: diff --git a/shifter/engine/provisioner/raes_gcp_apply.py b/shifter/engine/provisioner/raes_gcp_apply.py index 104a414c1..5e88a0c03 100644 --- a/shifter/engine/provisioner/raes_gcp_apply.py +++ b/shifter/engine/provisioner/raes_gcp_apply.py @@ -36,6 +36,7 @@ from gcp_range_cell_ops import _get_or_none from gcp_range_cell_outputs import InstanceCredentials, instance_output, subnet_outputs from gcp_range_cell_resources import instance_resource +from gcp_range_cell_shared_nat import assert_shared_nat_capacity from gcp_range_cell_types import GceEgressPolicy, InstancePlan, RangeCellPlan, ResourceDict from gcp_range_cells import ( _assert_preconfigured_host_binding, @@ -210,6 +211,7 @@ def _provision_raes_resources( authored account credential installed and verified on the guest, so a declared endpoint never appears with a credential that was never realized. """ + assert_shared_nat_capacity(plan, runtime.clients) if plan["manage_network"]: _record_created(created, "network", plan["network"]["name"], _ensure_network(plan, runtime.clients)) for subnet in plan["subnets"]: diff --git a/shifter/engine/provisioner/raes_gcp_destroy.py b/shifter/engine/provisioner/raes_gcp_destroy.py index 21a90585f..150974ac4 100644 --- a/shifter/engine/provisioner/raes_gcp_destroy.py +++ b/shifter/engine/provisioner/raes_gcp_destroy.py @@ -23,6 +23,7 @@ from gcp_range_cell_firewall import public_web_firewall_name from gcp_range_cell_model_broker import broker_firewall_name from gcp_range_cell_ops import _delete_resource +from gcp_range_cell_shared_nat import remove_shared_nat from gcp_range_cell_types import InstancePlan, RangeCellPlan from raes_account_credentials import ( RaesAccountCredentialOps, @@ -157,6 +158,7 @@ def _instance_accounts(raes_plan: RaesPlan, instance: InstancePlan) -> tuple[Rae def _destroy_network_resources(plan: RangeCellPlan, clients: GCEClients) -> None: """Delete the range-owned router/NAT, firewalls, subnets, and an owned network in order.""" + remove_shared_nat(plan, clients) # The range-owned Cloud Router (carrying the Cloud NAT) references this range's # subnets, so it is torn down before them (PLAT-238). Absent for a `none` range. router_nat = plan.get("router_nat") diff --git a/shifter/engine/provisioner/raes_gcp_plan.py b/shifter/engine/provisioner/raes_gcp_plan.py index 1246fc536..e3ae84240 100644 --- a/shifter/engine/provisioner/raes_gcp_plan.py +++ b/shifter/engine/provisioner/raes_gcp_plan.py @@ -42,6 +42,7 @@ _subnet_tag, _subnetwork_self_link, range_router_nat_plan, + shared_router_nat_plan, ) from gcp_range_cell_plan import _range_labels from gcp_range_cell_types import ( @@ -194,10 +195,14 @@ def build_raes_range_cell_plan( # A non-`none` range owns an explicit Cloud Router + NAT scoped to its subnets; # a `none` (zero-egress) range omits it so its subnets carry no NAT path. if (resolved_options.egress_policy.mode or "status-quo").strip().lower() != "none": - plan["router_nat"] = cast( - RouterNatPlan, - range_router_nat_plan(range_id, [subnet["self_link"] for subnet in subnet_plans]), - ) + if manage_network: + plan["router_nat"] = cast( + RouterNatPlan, range_router_nat_plan(range_id, [subnet["self_link"] for subnet in subnet_plans]) + ) + else: + plan["shared_nat"] = cast( + RouterNatPlan, shared_router_nat_plan(network_name, [subnet["self_link"] for subnet in subnet_plans]) + ) return plan diff --git a/shifter/engine/provisioner/tests/test_gcp_range_cells.py b/shifter/engine/provisioner/tests/test_gcp_range_cells.py index e0af26c02..d5b428097 100644 --- a/shifter/engine/provisioner/tests/test_gcp_range_cells.py +++ b/shifter/engine/provisioner/tests/test_gcp_range_cells.py @@ -56,6 +56,13 @@ def _stub_range_data_for_pool_slot(monkeypatch): stub = MagicMock(return_value={"vpn_gateway_pool_slot": _TEST_VPN_GATEWAY_POOL_SLOT}) monkeypatch.setattr("gcp_range_cells.get_range_data_by_request_id", stub, raising=False) monkeypatch.setattr("gcp_range_cell_destroy.get_range_data_by_request_id", stub, raising=False) + # Shared NAT holds a database advisory lock; keep these Compute lifecycle + # tests at the external DB boundary, not a first-party service mock. + monkeypatch.setenv("DB_HOST", "localhost") + monkeypatch.setenv("DB_USER", "test") + monkeypatch.setenv("DB_NAME", "test") + monkeypatch.setenv("DB_PASSWORD", "test-only") + monkeypatch.setattr("provisioner_db.psycopg.connect", lambda **_kwargs: MagicMock()) return stub @@ -238,12 +245,42 @@ def service(): op_service = MagicMock() op_service.wait.return_value = SimpleNamespace(status="DONE") + routers = service() + router_state = {} + + def router_get(**kwargs): + name = kwargs["router"] + if name in router_state: + return router_state[name] + if exists: + return {"name": name, "network": "projects/test-project/global/networks/shared-range", "nats": []} + raise NotFound() + + def router_insert(**kwargs): + body = kwargs["router_resource"] + router_state[body["name"]] = body + return SimpleNamespace(name="op") + + def router_patch(**kwargs): + name = kwargs["router"] + router_state[name] = {**router_state[name], **kwargs["router_resource"]} + return SimpleNamespace(name="op") + + def router_delete(**kwargs): + router_state.pop(kwargs["router"], None) + return SimpleNamespace(name="op") + + routers.get.side_effect = router_get + routers.list.side_effect = lambda **_kwargs: list(router_state.values()) + routers.insert.side_effect = router_insert + routers.patch.side_effect = router_patch + routers.delete.side_effect = router_delete return SimpleNamespace( networks=service(), subnetworks=service(), firewalls=service(), addresses=service(), - routers=service(), + routers=routers, instances=service(), global_operations=op_service, region_operations=op_service, @@ -677,6 +714,30 @@ def test_apply_shared_vpc_skips_network_create(mocker): clients.subnetworks.insert.assert_called() +def test_shared_nat_capacity_refusal_precedes_subnet_creation(mocker): + clients = _mock_clients(exists=False) + secret_ops, _ = _mock_secret_ops(mocker) + vertex_ops, _ = _mock_vertex_ops(mocker) + clients.routers.list.return_value = [ + {"name": f"other-{index}", "network": "projects/test-project/global/networks/shared-range", "nats": []} + for index in range(5) + ] + clients.routers.list.side_effect = None + + with pytest.raises(RuntimeError, match="shared-nat-router-capacity-exhausted"): + apply_range_cell( + "req-123", + _variables(), + config=_shared_vpc_config(), + clients=clients, + secret_ops=secret_ops, + vertex_ops=vertex_ops, + cleanup_range_cell=lambda *_args: None, + ) + + clients.subnetworks.insert.assert_not_called() + + def test_destroy_shared_vpc_skips_network_delete(mocker): clients = _mock_clients(exists=True) secret_ops, _ = _mock_secret_ops(mocker) diff --git a/shifter/engine/provisioner/tests/test_gcp_shared_nat.py b/shifter/engine/provisioner/tests/test_gcp_shared_nat.py new file mode 100644 index 000000000..ae83bfa96 --- /dev/null +++ b/shifter/engine/provisioner/tests/test_gcp_shared_nat.py @@ -0,0 +1,262 @@ +"""Shared-VPC NAT allocation must scale without widening subnet scope.""" + +from __future__ import annotations + +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +import gcp_range_cell_shared_nat as shared_nat +from gcp_range_cell_naming import shared_router_nat_plan +from gcp_range_cell_shared_nat import ( + allocate_explicit_subnets, + assert_shared_nat_capacity, + ensure_shared_nat, + remove_shared_nat, +) + + +def test_shared_gateway_name_fits_provider_limit_for_long_network_name(): + nat = shared_router_nat_plan("a" * 63, ["projects/p/regions/r/subnetworks/range-1"]) + gateway = allocate_explicit_subnets({}, nat["subnetwork_self_links"], nat_name=nat["nat_name"])[0] + + assert len(gateway["name"]) <= 63 + + +def test_twenty_two_two_subnet_ranges_fit_one_gateway_without_global_nat(): + subnets = [f"projects/p/regions/r/subnetworks/range-{seat}-{side}" for seat in range(22) for side in range(2)] + + gateways = allocate_explicit_subnets({}, subnets, nat_name="shifter-shared-nat") + + assert len(gateways) == 1 + assert gateways[0]["source_subnetwork_ip_ranges_to_nat"] == "LIST_OF_SUBNETWORKS" + assert [item["name"] for item in gateways[0]["subnetworks"]] == subnets + + +def test_gateway_sharding_stops_at_explicit_subnet_limit(): + subnets = [f"projects/p/regions/r/subnetworks/range-{index}" for index in range(51)] + + gateways = allocate_explicit_subnets({}, subnets, nat_name="shifter-shared-nat") + + assert [len(gateway["subnetworks"]) for gateway in gateways] == [50, 1] + + +def test_capacity_exhaustion_refuses_before_any_widening(): + subnets = [f"projects/p/regions/r/subnetworks/range-{index}" for index in range(2501)] + + with pytest.raises(RuntimeError, match="shared-nat-capacity-exhausted"): + allocate_explicit_subnets({}, subnets, nat_name="shifter-shared-nat") + + +def test_gateway_reuse_never_overwrites_an_occupied_sparse_slot(): + existing = { + "shifter-shared-nat-1": [f"projects/p/regions/r/subnetworks/a-{index}" for index in range(50)], + "shifter-shared-nat-2": [f"projects/p/regions/r/subnetworks/b-{index}" for index in range(50)], + } + + gateways = allocate_explicit_subnets( + existing, ["projects/p/regions/r/subnetworks/new"], nat_name="shifter-shared-nat" + ) + + assert {gateway["name"] for gateway in gateways} == set(existing) | {"shifter-shared-nat-0"} + assert sum(len(gateway["subnetworks"]) for gateway in gateways) == 101 + + +def test_provider_full_self_link_replay_does_not_duplicate_relative_subnet(): + relative = "projects/p/regions/r/subnetworks/range-1" + existing = {"shifter-shared-nat-0": [f"https://www.googleapis.com/compute/v1/{relative}"]} + + gateways = allocate_explicit_subnets(existing, [relative], nat_name="shifter-shared-nat") + + assert len(gateways[0]["subnetworks"]) == 1 + + +def _plan_for_capacity(*, egress: bool = True) -> dict: + plan = { + "project_id": "p", + "region": "r", + "range_id": 7, + "manage_network": False, + "network": {"self_link": "projects/p/global/networks/shared"}, + "subnets": [{"self_link": "projects/p/regions/r/subnetworks/range-7"}], + } + if egress: + plan["shared_nat"] = { + "router_name": "shifter-shared-nat-router", + "nat_name": "shifter-shared-nat", + "subnetwork_self_links": [plan["subnets"][0]["self_link"]], + } + return plan + + +def test_none_range_is_refused_if_a_foreign_all_subnet_nat_exists(monkeypatch): + monkeypatch.setattr(shared_nat, "_regional_nat_lock", lambda _plan: nullcontext()) + router = { + "name": "foreign-router", + "network": "projects/p/global/networks/shared", + "nats": [{"source_subnetwork_ip_ranges_to_nat": "ALL_SUBNETWORKS_ALL_IP_RANGES"}], + } + clients = SimpleNamespace(routers=SimpleNamespace(list=lambda **_kwargs: [router])) + + with pytest.raises(RuntimeError, match="shared-nat-unbounded-or-unknown-scope"): + assert_shared_nat_capacity(_plan_for_capacity(egress=False), clients) + + +def test_router_limit_is_checked_before_creating_range_subnets(monkeypatch): + class NotFound(Exception): + pass + + monkeypatch.setattr(shared_nat, "_regional_nat_lock", lambda _plan: nullcontext()) + routers = [ + {"name": f"other-{index}", "network": "projects/p/global/networks/shared", "nats": []} for index in range(5) + ] + + def get(**_kwargs): + raise NotFound() + + clients = SimpleNamespace( + routers=SimpleNamespace(list=lambda **_kwargs: routers, get=get), + google_exceptions=SimpleNamespace(NotFound=NotFound), + ) + + with pytest.raises(RuntimeError, match="shared-nat-router-capacity-exhausted"): + assert_shared_nat_capacity(_plan_for_capacity(), clients) + + +def test_existing_bridge_subnet_replay_preserves_terraform_nat(monkeypatch): + class NotFound(Exception): + pass + + monkeypatch.setattr(shared_nat, "_regional_nat_lock", lambda _plan: nullcontext()) + plan = _plan_for_capacity() + plan["network"]["name"] = "shared" + subnet = plan["subnets"][0]["self_link"] + bridge = { + "name": "shared-nat", + "network": plan["network"]["self_link"], + "nats": [ + { + "name": "shared-nat", + "nat_ip_allocate_option": "MANUAL_ONLY", + "source_subnetwork_ip_ranges_to_nat": "LIST_OF_SUBNETWORKS", + "subnetworks": [{"name": f"https://www.googleapis.com/compute/v1/{subnet}"}], + } + ], + } + + def get_router(**_kwargs): + raise NotFound() + + clients = SimpleNamespace( + routers=SimpleNamespace(list=lambda **_kwargs: [bridge], get=get_router), + subnetworks=SimpleNamespace(get=lambda **_kwargs: {"network": plan["network"]["self_link"]}), + google_exceptions=SimpleNamespace(NotFound=NotFound), + ) + + assert_shared_nat_capacity(plan, clients) + ensure_shared_nat(plan, clients) + with pytest.raises(RuntimeError, match="shared-nat-migration-bridge-attached"): + remove_shared_nat(plan, clients) + + +def test_bridge_cannot_enroll_a_new_range_subnet(monkeypatch): + class NotFound(Exception): + pass + + monkeypatch.setattr(shared_nat, "_regional_nat_lock", lambda _plan: nullcontext()) + plan = _plan_for_capacity() + subnet = plan["subnets"][0]["self_link"] + bridge = { + "name": "shared-nat", + "network": plan["network"]["self_link"], + "nats": [ + { + "name": "shared-nat", + "nat_ip_allocate_option": "MANUAL_ONLY", + "source_subnetwork_ip_ranges_to_nat": "LIST_OF_SUBNETWORKS", + "subnetworks": [{"name": subnet}], + } + ], + } + + def missing_subnet(**_kwargs): + raise NotFound() + + clients = SimpleNamespace( + routers=SimpleNamespace(list=lambda **_kwargs: [bridge]), + subnetworks=SimpleNamespace(get=missing_subnet), + google_exceptions=SimpleNamespace(NotFound=NotFound), + ) + + with pytest.raises(RuntimeError, match="shared-nat-foreign-overlap"): + assert_shared_nat_capacity(plan, clients) + + +def test_convergent_enrollment_and_removal_preserve_other_ranges(monkeypatch): + class NotFound(Exception): + pass + + routers = {} + service = MagicMock() + + def get(**kwargs): + try: + return routers[kwargs["router"]] + except KeyError: + raise NotFound() from None + + def insert(**kwargs): + body = kwargs["router_resource"] + routers[body["name"]] = body + return None + + def patch(**kwargs): + routers[kwargs["router"]] = {**routers[kwargs["router"]], **kwargs["router_resource"]} + return None + + def delete(**kwargs): + routers.pop(kwargs["router"]) + return None + + service.get.side_effect = get + service.insert.side_effect = insert + service.patch.side_effect = patch + service.delete.side_effect = delete + clients = SimpleNamespace(routers=service, google_exceptions=SimpleNamespace(NotFound=NotFound)) + monkeypatch.setattr(shared_nat, "_regional_nat_lock", lambda _plan: nullcontext()) + monkeypatch.setattr(shared_nat, "_wait_for_operation", lambda *_args: None) + + def plan(range_id): + link = f"projects/p/regions/r/subnetworks/range-{range_id}" + return { + "project_id": "p", + "region": "r", + "range_id": range_id, + "network": {"self_link": "projects/p/global/networks/shared"}, + "shared_nat": { + "router_name": "shifter-shared-nat-router", + "nat_name": "shifter-shared-nat", + "subnetwork_self_links": [link], + }, + } + + ensure_shared_nat(plan(1), clients) + ensure_shared_nat(plan(2), clients) + ensure_shared_nat(plan(1), clients) + + assert service.insert.call_count == 1 + assert service.patch.call_count == 1 + assert [s["name"] for s in routers["shifter-shared-nat-router"]["nats"][0]["subnetworks"]] == [ + "projects/p/regions/r/subnetworks/range-1", + "projects/p/regions/r/subnetworks/range-2", + ] + + remove_shared_nat(plan(1), clients) + assert "shifter-shared-nat-router" in routers + assert [s["name"] for s in routers["shifter-shared-nat-router"]["nats"][0]["subnetworks"]] == [ + "projects/p/regions/r/subnetworks/range-2" + ] + remove_shared_nat(plan(2), clients) + assert not routers diff --git a/shifter/engine/provisioner/tests/test_raes_gcp_apply.py b/shifter/engine/provisioner/tests/test_raes_gcp_apply.py index 6b42780f1..c34a6021e 100644 --- a/shifter/engine/provisioner/tests/test_raes_gcp_apply.py +++ b/shifter/engine/provisioner/tests/test_raes_gcp_apply.py @@ -50,6 +50,15 @@ class _NotFound(Exception): """Fake Google NotFound exception.""" +@pytest.fixture(autouse=True) +def _fake_advisory_lock_database(monkeypatch): + monkeypatch.setenv("DB_HOST", "localhost") + monkeypatch.setenv("DB_USER", "test") + monkeypatch.setenv("DB_NAME", "test") + monkeypatch.setenv("DB_PASSWORD", "test-only") + monkeypatch.setattr("provisioner_db.psycopg.connect", lambda **_kwargs: MagicMock()) + + def _config(network_mode: str = "vpc-per-range") -> GCERangeCellConfig: return GCERangeCellConfig( project_id="proj-1", @@ -114,12 +123,42 @@ def service(insert_error: Exception | None = None): op_service = MagicMock() op_service.wait.return_value = SimpleNamespace(status="DONE") + routers = service() + router_state = {} + + def router_get(**kwargs): + name = kwargs["router"] + if name in router_state: + return router_state[name] + if exists: + return {"name": name, "network": "projects/proj-1/global/networks/shared", "nats": []} + raise _NotFound() + + def router_insert(**kwargs): + body = kwargs["router_resource"] + router_state[body["name"]] = body + return SimpleNamespace(name="op") + + def router_patch(**kwargs): + name = kwargs["router"] + router_state[name] = {**router_state[name], **kwargs["router_resource"]} + return SimpleNamespace(name="op") + + def router_delete(**kwargs): + router_state.pop(kwargs["router"], None) + return SimpleNamespace(name="op") + + routers.get.side_effect = router_get + routers.list.side_effect = lambda **_kwargs: list(router_state.values()) + routers.insert.side_effect = router_insert + routers.patch.side_effect = router_patch + routers.delete.side_effect = router_delete return SimpleNamespace( networks=service(), subnetworks=service(), firewalls=service(), addresses=service(), - routers=service(), + routers=routers, instances=service(instance_insert_error), global_operations=op_service, region_operations=op_service, @@ -1332,6 +1371,7 @@ def test_shared_vpc_destroy_keeps_network(self): ), ) assert not clients.networks.delete.called + assert clients.routers.delete.called def test_deletes_every_per_instance_authored_account_secret(self): account = RaesPlanAccount(username="alice", target_address="node.web", auth_method="key") diff --git a/shifter/engine/provisioner/tests/test_raes_gcp_plan.py b/shifter/engine/provisioner/tests/test_raes_gcp_plan.py index 288df9778..68f84cfdc 100644 --- a/shifter/engine/provisioner/tests/test_raes_gcp_plan.py +++ b/shifter/engine/provisioner/tests/test_raes_gcp_plan.py @@ -830,3 +830,18 @@ def test_none_range_has_no_router_nat(self): egress_policy=GceEgressPolicy(mode="none"), ) assert "router_nat" not in plan + + def test_shared_vpc_uses_regional_explicit_nat_instead_of_range_router(self): + plan = build_raes_range_cell_plan( + "req-1", + 7, + _plan((_node(),), (_network(),)), + _resolver(), + _config(network_mode="shared-vpc", network_id="projects/proj-1/global/networks/ranges"), + egress_policy=GceEgressPolicy(mode="status-quo"), + allocated_network_cidrs=(("net.a", "10.90.1.0/24"),), + ) + + assert "router_nat" not in plan + assert plan["shared_nat"]["router_name"] == "shifter-ranges-nat-router" + assert plan["shared_nat"]["subnetwork_self_links"] == [subnet["self_link"] for subnet in plan["subnets"]] diff --git a/shifter/shifter_platform/cms/services/__init__.py b/shifter/shifter_platform/cms/services/__init__.py index 8c780fac6..27b3eed09 100644 --- a/shifter/shifter_platform/cms/services/__init__.py +++ b/shifter/shifter_platform/cms/services/__init__.py @@ -168,7 +168,11 @@ list_mission_control_range_history, list_ranges, ) -from ._range_reassign import range_owner_reassignment_available, reassign_range_owner +from ._range_reassign import ( + range_egress_compatible_with_event, + range_owner_reassignment_available, + reassign_range_owner, +) from ._range_resume import resume_range, resume_range_by_request_id from ._range_vpn import ( CtfOpenVpnProfileConflict, @@ -357,6 +361,7 @@ "project_scenario_images", "project_scenario_model_demands", "range_credential_scope", + "range_egress_compatible_with_event", "range_owner_reassignment_available", "range_status_changed", "reassign_range_owner", diff --git a/shifter/shifter_platform/cms/services/_non_user_range_launch.py b/shifter/shifter_platform/cms/services/_non_user_range_launch.py index a879fdd90..d00bec151 100644 --- a/shifter/shifter_platform/cms/services/_non_user_range_launch.py +++ b/shifter/shifter_platform/cms/services/_non_user_range_launch.py @@ -28,7 +28,7 @@ from typing import TYPE_CHECKING from cms.exceptions import CMSError -from cms.services._raes_range_create import dispatch_range_launch +from cms.services._raes_range_dispatch import dispatch_range_launch from cms.services._range_launch_common import LaunchOptions from shared.range_instantiation_policy import POLICY_DENIAL_CODE, InstantiationPurpose diff --git a/shifter/shifter_platform/cms/services/_raes_range_create.py b/shifter/shifter_platform/cms/services/_raes_range_create.py index e22aa08fe..f09ae1a62 100644 --- a/shifter/shifter_platform/cms/services/_raes_range_create.py +++ b/shifter/shifter_platform/cms/services/_raes_range_create.py @@ -113,7 +113,15 @@ def _dispatch_raes_package( ) -def _audit_raes_range_provision(request_id: UUID, scenario: str, user: User, range_source: RangeSource) -> None: +def _audit_raes_range_provision( + request_id: UUID, + scenario: str, + user: User, + range_source: RangeSource, + *, + egress_policy_workspace_id: int, + egress_mode: str, +) -> None: """Write the audit-log entry for a successful RAES-native launch.""" instance = RangeInstance.objects.filter(request__request_id=request_id).first() lease_state: dict[str, object] = {} @@ -137,6 +145,8 @@ def _audit_raes_range_provision(request_id: UUID, scenario: str, user: User, ran "scenario": scenario, "provisioning": "raes-native", "range_source": range_source.value, + "egress_policy_workspace_id": egress_policy_workspace_id, + "egress_mode": egress_mode, **lease_state, }, request_id=str(request_id), @@ -215,6 +225,7 @@ def _create_raes_native_range_impl( # NOSONAR -- mirrors the stable launch serv model_launch_scope: ModelLaunchScope | None = None, model_sources: dict | None = None, content_authorizer: User | None = None, + ctf_policy_workspace_id: int | None = None, ) -> RangeContext: """Shared RAES creation body, parameterized by minted launch authority. @@ -291,7 +302,15 @@ def _persist(cms_request: Request) -> RangeInstance: from cms.services._range_workspace import resolve_effective_egress_mode - egress_mode = resolve_effective_egress_mode(workspace_id) + policy_workspace_id = workspace_id + if ctf_policy_workspace_id is not None: + from cms.services._range_workspace import authorize_ctf_policy_workspace + + if range_source is not RangeSource.CTF or content_authorizer is None: + raise CMSError("Selected workspace is not available") + authorize_ctf_policy_workspace(content_authorizer, ctf_policy_workspace_id) + policy_workspace_id = ctf_policy_workspace_id + egress_mode = resolve_effective_egress_mode(policy_workspace_id) from shared.range_instantiation_policy import assert_range_backend_egress_supported try: @@ -355,14 +374,28 @@ def _persist(cms_request: Request) -> RangeInstance: enforced_deadline=enforced_deadline, model_launch_scope=model_launch_scope, model_sources=model_sources, + policy_workspace_id=ctf_policy_workspace_id, ) ) if claimed_request_id is not None: - _audit_raes_range_provision(claimed_request_id, scenario, user, range_source) + _audit_raes_range_provision( + claimed_request_id, + scenario, + user, + range_source, + egress_policy_workspace_id=policy_workspace_id, + egress_mode=egress_mode, + ) return _build_raes_range_context(claimed_request_id, scenario, user) _request_id, _cms_request, range_instance, egress_mode = _reserve_active_range_slot( - user, range_source, _persist, workspace_id, request_id + user, + range_source, + _persist, + workspace_id, + request_id, + policy_workspace_id=ctf_policy_workspace_id, + policy_actor=content_authorizer, ) try: @@ -404,7 +437,14 @@ def _persist(cms_request: Request) -> RangeInstance: release_workspace_concurrent_range(workspace_id, request_id) raise - _audit_raes_range_provision(request_id, scenario, user, range_source) + _audit_raes_range_provision( + request_id, + scenario, + user, + range_source, + egress_policy_workspace_id=policy_workspace_id, + egress_mode=egress_mode, + ) return _build_raes_range_context(request_id, scenario, user) @@ -419,6 +459,7 @@ def create_range_dispatch( # NOSONAR -- stable cross-service facade retained fo model_launch_scope: ModelLaunchScope | None = None, model_sources: dict | None = None, content_authorizer: User | None = None, + ctf_policy_workspace_id: int | None = None, ) -> RangeContext: """Launch a registered RAES scenario through the authoritative path. @@ -430,6 +471,8 @@ def create_range_dispatch( # NOSONAR -- stable cross-service facade retained fo threaded to whichever create path runs. Server-derived callers (e.g. the CTF bridge) omit it, so their ranges bind to the launcher's personal workspace. """ + from cms.services._raes_range_dispatch import dispatch_range_launch + return dispatch_range_launch( user, scenario, @@ -443,38 +486,6 @@ def create_range_dispatch( # NOSONAR -- stable cross-service facade retained fo model_launch_scope=model_launch_scope, model_sources=model_sources, content_authorizer=content_authorizer, + ctf_policy_workspace_id=ctf_policy_workspace_id, ), ) - - -def dispatch_range_launch( - user: User, - scenario: str, - *, - range_source: RangeSource | None, - instantiation_purpose: InstantiationPurpose, - options: LaunchOptions, -) -> RangeContext: - """Shared RAES launch body, parameterized by minted launch authority. - - Not a product facade. Internal to - the CMS create seam -- ``cms.services`` exports the two facades that wrap it, - never this function. ``options`` bundles the optional launch-shaping inputs - (see :class:`cms.services._range_launch_common.LaunchOptions`). - """ - # RAES participant access is authored in the package and persisted as the - # compiled participant-access sidecar. The server-derived CTF cleanup time - # bounds the range lease; it does not mint an OpenVPN capability or alter the - # RAES plan. - return _create_raes_native_range_impl( - user, - scenario, - range_source=range_source, - instantiation_purpose=instantiation_purpose, - workspace_uuid=options.workspace_uuid, - enforced_deadline=options.remote_access_teardown_at, - model_admission_subject=options.model_admission_subject, - model_launch_scope=options.model_launch_scope, - model_sources=options.model_sources, - content_authorizer=options.content_authorizer, - ) diff --git a/shifter/shifter_platform/cms/services/_raes_range_dispatch.py b/shifter/shifter_platform/cms/services/_raes_range_dispatch.py new file mode 100644 index 000000000..bc3028e19 --- /dev/null +++ b/shifter/shifter_platform/cms/services/_raes_range_dispatch.py @@ -0,0 +1,40 @@ +"""Internal dispatch facade for RAES range launches.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from cms.services._range_launch_common import LaunchOptions +from shared.range_instantiation_policy import InstantiationPurpose + +if TYPE_CHECKING: + from django.contrib.auth.models import User + + from shared.enums import RangeSource + from shared.schemas.range import RangeContext + + +def dispatch_range_launch( + user: User, + scenario: str, + *, + range_source: RangeSource | None, + instantiation_purpose: InstantiationPurpose, + options: LaunchOptions, +) -> RangeContext: + """Launch with the authority and policy prepared by the caller.""" + from cms.services._raes_range_create import _create_raes_native_range_impl + + return _create_raes_native_range_impl( + user, + scenario, + range_source=range_source, + instantiation_purpose=instantiation_purpose, + workspace_uuid=options.workspace_uuid, + enforced_deadline=options.remote_access_teardown_at, + model_admission_subject=options.model_admission_subject, + model_launch_scope=options.model_launch_scope, + model_sources=options.model_sources, + content_authorizer=options.content_authorizer, + ctf_policy_workspace_id=options.ctf_policy_workspace_id, + ) diff --git a/shifter/shifter_platform/cms/services/_range_launch_common.py b/shifter/shifter_platform/cms/services/_range_launch_common.py index 998a805d8..160fbed02 100644 --- a/shifter/shifter_platform/cms/services/_range_launch_common.py +++ b/shifter/shifter_platform/cms/services/_range_launch_common.py @@ -52,6 +52,7 @@ class LaunchOptions: # CTF ranges belong to a participant, while private scenario visibility is # authorized by the event owner who selected the tenant pack. content_authorizer: User | None = None + ctf_policy_workspace_id: int | None = None def _audit_log_call(**kwargs: Any) -> None: # NOSONAR @@ -153,6 +154,9 @@ def _reserve_active_range_slot( persist_instance: Callable[[Request], RangeInstance], workspace_id: int, request_id: UUID | None = None, + *, + policy_workspace_id: int | None = None, + policy_actor: User | None = None, ) -> tuple[UUID, Request, RangeInstance, str]: """Atomically reauthorize scope, admit the workspace quota, and reserve the slot.""" from uuid import uuid4 @@ -168,13 +172,21 @@ def _reserve_active_range_slot( quota_audit = WorkspaceQuotaAuditContext(actor_type="user", actor_id=getattr(user, "id", None)) try: with transaction.atomic(): + if policy_workspace_id is not None and policy_actor is not None and policy_workspace_id < workspace_id: + from cms.services._range_workspace import reauthorize_ctf_policy_workspace_locked + + reauthorize_ctf_policy_workspace_locked(policy_actor, policy_workspace_id) reauthorize_launch_workspace_locked(user, workspace_id) + if policy_workspace_id is not None and policy_actor is not None and policy_workspace_id >= workspace_id: + from cms.services._range_workspace import reauthorize_ctf_policy_workspace_locked + + reauthorize_ctf_policy_workspace_locked(policy_actor, policy_workspace_id) # Concurrent-range quota is evaluated under the same workspace mutex and # the open reservation is committed with the CMS reservation, so an # active-range collision or any persistence failure rolls both back # together (ADR-046-R10). The pre-minted request UUID is the key. reserve_workspace_concurrent_range(workspace_id, correlation_id, quota_audit) - egress_mode = resolve_effective_egress_mode_locked(workspace_id) + egress_mode = resolve_effective_egress_mode_locked(policy_workspace_id or workspace_id) cms_request = _create_cms_request(user, workspace_id, correlation_id) range_instance = persist_instance(cms_request) _set_range_instance_status(range_instance, ResourceStatus.PROVISIONING) diff --git a/shifter/shifter_platform/cms/services/_range_reassign.py b/shifter/shifter_platform/cms/services/_range_reassign.py index ebb7a45ae..8ffd9f028 100644 --- a/shifter/shifter_platform/cms/services/_range_reassign.py +++ b/shifter/shifter_platform/cms/services/_range_reassign.py @@ -42,6 +42,28 @@ def range_owner_reassignment_available(range_instance_pk: int) -> bool: return _cs.engine_range_owner_reassignment_available(instance.request.request_id) +def range_egress_compatible_with_event( + range_instance_pk: int, event_owner: User, event_workspace_id: int | None +) -> bool: + """Compare a spare's pinned posture with the current authorized event policy.""" + from cms.services._range_workspace import ( + reauthorize_ctf_policy_workspace_locked, + resolve_effective_egress_mode_locked, + ) + from engine.services import get_pinned_range_egress_mode_by_request + + instance = RangeInstance.objects.select_related("request").filter(pk=range_instance_pk).first() + if instance is None or instance.request is None: + return False + if event_workspace_id is None: + # Personal-scope events have no separate event policy source. + return True + reauthorize_ctf_policy_workspace_locked(event_owner, event_workspace_id) + current_mode = resolve_effective_egress_mode_locked(event_workspace_id) + pinned_mode = get_pinned_range_egress_mode_by_request(instance.request.request_id) + return pinned_mode is not None and pinned_mode == current_mode + + def _engine_rebind_range_workspace_call( request_id: uuid.UUID, *, expected_workspace_id: int, new_workspace_id: int ) -> RangeWorkspaceRebindOutcome: # NOSONAR diff --git a/shifter/shifter_platform/cms/services/_range_workspace.py b/shifter/shifter_platform/cms/services/_range_workspace.py index d1c769a1a..9ee8de83a 100644 --- a/shifter/shifter_platform/cms/services/_range_workspace.py +++ b/shifter/shifter_platform/cms/services/_range_workspace.py @@ -29,6 +29,26 @@ _LAUNCH_SCOPE_DENIED = "Selected workspace is not available" +def authorize_ctf_policy_workspace(actor: User, workspace_id: int) -> None: + """Authorize the event owner to use a server-derived CTF policy source.""" + from workspaces.services import WorkspaceAuthorizationError, authorize_bound_workspace + + try: + authorize_bound_workspace(actor, workspace_id, WorkspaceOperation.USE_CTF_COMMUNICATIONS) + except WorkspaceAuthorizationError as exc: + raise WorkspaceLaunchDenied(_LAUNCH_SCOPE_DENIED) from exc + + +def reauthorize_ctf_policy_workspace_locked(actor: User, workspace_id: int) -> None: + """Recheck the event policy grant under the workspace mutation mutex.""" + from workspaces.services import WorkspaceAuthorizationError, authorize_launch_workspace_locked + + try: + authorize_launch_workspace_locked(actor, workspace_id, WorkspaceOperation.USE_CTF_COMMUNICATIONS) + except WorkspaceAuthorizationError as exc: + raise WorkspaceLaunchDenied(_LAUNCH_SCOPE_DENIED) from exc + + def resolve_launch_workspace(user: User, workspace_uuid: str | uuid.UUID | None = None) -> int: """Resolve and authorize the workspace scope a launch by ``user`` belongs to. diff --git a/shifter/shifter_platform/cms/services/_warm_pool_claim.py b/shifter/shifter_platform/cms/services/_warm_pool_claim.py index 26b199ff6..6b79fa590 100644 --- a/shifter/shifter_platform/cms/services/_warm_pool_claim.py +++ b/shifter/shifter_platform/cms/services/_warm_pool_claim.py @@ -102,6 +102,7 @@ class WarmClaimRequest: enforced_deadline: datetime | None = None model_launch_scope: ModelLaunchScope | None = None model_sources: dict | None = None + policy_workspace_id: int | None = None @dataclass(frozen=True) @@ -261,8 +262,14 @@ def attempt_warm_claim(request: WarmClaimRequest, override: WarmPoolOverride | N from engine.services import enqueue_range_activation from shared.warm_pool.metrics import CLAIM_HIT, emit_claim_outcome - candidates = _resolve_claim_candidates(request, override) - if not candidates or not _can_claim_base_range(request): + # Existing warm generations are not bound to an event's policy workspace. + # An event-policy launch must take the cold reservation path, which pins that + # policy under the workspace lock before dispatch. + if ( + request.policy_workspace_id is not None + or not (candidates := _resolve_claim_candidates(request, override)) + or not _can_claim_base_range(request) + ): return None outcome = _run_atomic_claim(request, candidates) if outcome is None: diff --git a/shifter/shifter_platform/cms/services/_warm_pool_reconcile.py b/shifter/shifter_platform/cms/services/_warm_pool_reconcile.py index d014e6c94..b43b9f3a3 100644 --- a/shifter/shifter_platform/cms/services/_warm_pool_reconcile.py +++ b/shifter/shifter_platform/cms/services/_warm_pool_reconcile.py @@ -387,7 +387,14 @@ def _persist(cms_request: Request) -> RangeInstance: return False _dispatch_raes_package(request_id, system_user, source, backend_admission, workspace_id, egress_mode) - _audit_raes_range_provision(request_id, bucket.scenario, system_user, WARM_RANGE_SOURCE) + _audit_raes_range_provision( + request_id, + bucket.scenario, + system_user, + WARM_RANGE_SOURCE, + egress_policy_workspace_id=workspace_id, + egress_mode=egress_mode, + ) return True except Exception: logger.exception("warm-pool: failed to prepare a generation for bucket=%s", bucket.id) diff --git a/shifter/shifter_platform/ctf/bridges.py b/shifter/shifter_platform/ctf/bridges.py index 8338a3d23..208554d50 100644 --- a/shifter/shifter_platform/ctf/bridges.py +++ b/shifter/shifter_platform/ctf/bridges.py @@ -84,6 +84,7 @@ class CTFRangeLaunchOptions: # CTF ranges belong to participants, while private pack visibility is # authorized by the event owner who selected the pack. content_authorizer: User | None = None + event_policy_workspace_id: int | None = None def cms_declare_event_capacity( @@ -208,6 +209,7 @@ def cms_create_range( model_admission_subject=options.model_admission_subject, model_launch_scope=options.model_launch_scope, content_authorizer=options.content_authorizer, + ctf_policy_workspace_id=options.event_policy_workspace_id, ) return RangeProvisionResult(request_id=result.request_id) @@ -410,6 +412,15 @@ def cms_range_owner_reassignment_available(range_instance_id: int) -> bool: return cms_services.range_owner_reassignment_available(range_instance_id) +def cms_range_egress_compatible_with_event( + range_instance_id: int, event_owner: User, event_workspace_id: int | None +) -> bool: + """Check a spare's pinned posture against its event policy before reservation.""" + import cms.services as cms_services + + return cms_services.range_egress_compatible_with_event(range_instance_id, event_owner, event_workspace_id) + + def cms_list_scenarios(user: User) -> list[tuple[str, str]]: """List CTF-event-selectable scenarios as (id, name) tuples for form choices. diff --git a/shifter/shifter_platform/ctf/services/range/provision.py b/shifter/shifter_platform/ctf/services/range/provision.py index 97ca61caf..24a03ea13 100644 --- a/shifter/shifter_platform/ctf/services/range/provision.py +++ b/shifter/shifter_platform/ctf/services/range/provision.py @@ -196,6 +196,7 @@ def provision_participant_range(participant_id: UUID) -> dict[str, Any]: event, participant.pk, participant_model_admission_subject(participant) ), content_authorizer=event.created_by, + event_policy_workspace_id=event.workspace_id, ), ) except Exception as e: diff --git a/shifter/shifter_platform/ctf/services/range/recovery_steps.py b/shifter/shifter_platform/ctf/services/range/recovery_steps.py index 2373eeda9..752b2bfa9 100644 --- a/shifter/shifter_platform/ctf/services/range/recovery_steps.py +++ b/shifter/shifter_platform/ctf/services/range/recovery_steps.py @@ -87,6 +87,7 @@ def _rebuild_replacement(participant: CTFParticipant, model_subject: OwnedRefere model_admission_subject=model_subject, model_launch_scope=project_event_model_scope(event, participant.pk, model_subject), content_authorizer=event.created_by, + event_policy_workspace_id=event.workspace_id, ), ) except Exception as e: @@ -121,7 +122,11 @@ def _claim_spare(participant: CTFParticipant, spare_range_instance_id: int | Non MUST run inside the caller's transaction (``select_for_update``); :func:`_ensure_spare_reserved` wraps this claim and the pointer write atomically. """ - from ctf.bridges import cms_get_range_status, cms_range_owner_reassignment_available + from ctf.bridges import ( + cms_get_range_status, + cms_range_egress_compatible_with_event, + cms_range_owner_reassignment_available, + ) event = participant.event candidates = ( @@ -142,6 +147,10 @@ def _claim_spare(participant: CTFParticipant, spare_range_instance_id: int | Non continue if not cms_range_owner_reassignment_available(candidate.range_instance_id): continue + if not cms_range_egress_compatible_with_event( + candidate.range_instance_id, event.created_by, event.workspace_id + ): + continue candidate.consumed_by = participant candidate.consumed_at = timezone.now() candidate.status = SpareRangeStatus.CONSUMED.value @@ -159,6 +168,18 @@ def _ensure_spare_reserved( BEFORE teardown, so a missing spare never strands the participant (#1018). """ if recovery.replacement_range_instance_id is not None: + from ctf.bridges import cms_range_egress_compatible_with_event + + event = participant.event + if not cms_range_egress_compatible_with_event( + recovery.replacement_range_instance_id, event.created_by, event.workspace_id + ): + raise _range_error( + "No compatible spare range available for reassignment", + category=RecoveryFailureCategory.NO_COMPATIBLE_SPARE, + participant_id=str(participant.pk), + event_id=str(event.pk), + ) return # Claim + record the pointer in ONE transaction: a crash between them rolls # both back, so a spare is never CONSUMED without a durable recovery pointer. diff --git a/shifter/shifter_platform/ctf/services/range/spares.py b/shifter/shifter_platform/ctf/services/range/spares.py index 3bf6c00cb..634f2c925 100644 --- a/shifter/shifter_platform/ctf/services/range/spares.py +++ b/shifter/shifter_platform/ctf/services/range/spares.py @@ -179,6 +179,7 @@ def _provision_one_spare(event: CTFEvent) -> CTFSpareRange: spare_id=spare.pk, ), content_authorizer=event.created_by, + event_policy_workspace_id=event.workspace_id, ), ) except Exception: diff --git a/shifter/shifter_platform/engine/services/__init__.py b/shifter/shifter_platform/engine/services/__init__.py index baa510811..e31ef7e14 100644 --- a/shifter/shifter_platform/engine/services/__init__.py +++ b/shifter/shifter_platform/engine/services/__init__.py @@ -160,6 +160,7 @@ RangeWorkspaceRebindOutcome, cancel_range_by_request, destroy_range_by_request, + get_pinned_range_egress_mode_by_request, range_owner_reassignment_available_by_request, reassign_range_owner_by_request, rebind_range_workspace_by_request, @@ -347,6 +348,7 @@ def reconcile_model_allocations(*, now: datetime | None = None, limit: int = 100 "get_openvpn_profile", "get_or_create_allocation_group", "get_owned_instance_request_ref", + "get_pinned_range_egress_mode_by_request", "get_range_membership", "get_range_model_policy_status", "get_range_pause_resume_capability", diff --git a/shifter/shifter_platform/engine/services/_range_by_request.py b/shifter/shifter_platform/engine/services/_range_by_request.py index 752c2c073..525dc45af 100644 --- a/shifter/shifter_platform/engine/services/_range_by_request.py +++ b/shifter/shifter_platform/engine/services/_range_by_request.py @@ -69,6 +69,13 @@ def range_owner_reassignment_available_by_request(request_id: UUID) -> bool: return Range.objects.filter(request__request_id=request_id, vpn_access_binding__isnull=True).exists() +def get_pinned_range_egress_mode_by_request(request_id: UUID) -> str | None: + """Project the immutable launch posture for a trusted CTF spare claim.""" + from engine.models import Range + + return Range.objects.filter(request__request_id=request_id).values_list("egress_mode", flat=True).first() + + def destroy_range_by_request(request_id: UUID) -> bool: """Tear down range infrastructure by request_id. diff --git a/shifter/shifter_platform/tests/cms/test_warm_pool_claim.py b/shifter/shifter_platform/tests/cms/test_warm_pool_claim.py index 3f196a0bb..47ba4baa9 100644 --- a/shifter/shifter_platform/tests/cms/test_warm_pool_claim.py +++ b/shifter/shifter_platform/tests/cms/test_warm_pool_claim.py @@ -71,6 +71,16 @@ def test_no_matching_bucket_cold_falls_back(self, monkeypatch): # Enabled + gce supported, but no bucket serves this scenario. assert attempt_warm_claim(_request("gce", "some-other-scenario")) is None + def test_event_policy_workspace_never_claims_unpinned_generation(self, monkeypatch): + from dataclasses import replace + + from django.conf import settings + + monkeypatch.setattr(settings, "WARM_POOL_POLICY", _ENABLED_GCE, raising=False) + request = replace(_request("gce", "example"), range_source=RangeSource.CTF, policy_workspace_id=2) + + assert attempt_warm_claim(request) is None + def test_narrowed_policy_excludes_an_unauthorized_candidate(self, monkeypatch): from django.conf import settings diff --git a/shifter/shifter_platform/tests/ctf/test_services/test_range_recovery.py b/shifter/shifter_platform/tests/ctf/test_services/test_range_recovery.py index cd7fc9c44..8c6e92f8b 100644 --- a/shifter/shifter_platform/tests/ctf/test_services/test_range_recovery.py +++ b/shifter/shifter_platform/tests/ctf/test_services/test_range_recovery.py @@ -40,7 +40,7 @@ RecoveryStrategy, SpareRangeStatus, ) -from ctf.exceptions import CTFNotFoundError, CTFRangeError, CTFValidationError +from ctf.exceptions import CTFRangeError from ctf.models import ( CTFAward, CTFBracket, @@ -51,7 +51,7 @@ CTFSubmission, CTFTeam, ) -from ctf.services.range.recovery import get_recovery_status, recover_participant_range +from ctf.services.range.recovery import recover_participant_range from ctf.services.range.spares import create_managed_spare_user from engine.models import Range as EngineRange from engine.models import Request as EngineRequest @@ -331,6 +331,39 @@ def test_old_range_access_denied_after_rebuild(self, rich_participant, organizer class TestReassignSpareRecovery: """``strategy=reassign_spare``: consume an event-scoped pooled spare.""" + @pytest.mark.django_db + def test_personal_scope_event_has_no_separate_egress_policy(self, event_with_scenario, organizer_user): + from ctf.bridges import cms_range_egress_compatible_with_event + + spare_user = create_managed_spare_user() + _spare, spare_range = _make_pooled_spare(event_with_scenario, owner=spare_user) + + assert cms_range_egress_compatible_with_event(spare_range.pk, organizer_user, None) + + @pytest.mark.django_db + def test_policy_changed_after_spare_provision_refuses_claim_before_old_range_teardown( + self, event_with_scenario, rich_participant, organizer_user + ): + from workspaces.models import Workspace + + participant, old_range = rich_participant + spare_user = create_managed_spare_user() + spare, spare_range = _make_pooled_spare(event_with_scenario, owner=spare_user) + Workspace.objects.filter(pk=event_with_scenario.workspace_id).update(egress_policy="none") + + with pytest.raises(CTFRangeError, match="No compatible spare"): + recover_participant_range( + participant.pk, + strategy=RecoveryStrategy.REASSIGN_SPARE.value, + operator=organizer_user, + spare_range_instance_id=spare_range.pk, + ) + + old_range.refresh_from_db() + spare.refresh_from_db() + assert old_range.status == ResourceStatus.READY.value + assert spare.consumed_by_id is None + @pytest.mark.django_db def test_reassign_spare_transfers_access_and_blocks_old_range( self, event_with_scenario, rich_participant, organizer_user @@ -638,6 +671,42 @@ class TestIdempotentRetry: attempted and no second audit row is written. """ + @pytest.mark.django_db + def test_reserved_spare_is_rechecked_after_event_policy_changes( + self, monkeypatch, event_with_scenario, rich_participant, second_participant_user, organizer_user + ): + from workspaces.models import Workspace + + participant, old_range = rich_participant + spare, spare_range = _make_pooled_spare(event_with_scenario, owner=second_participant_user) + + def fail_teardown(_request_id): + raise CloudTaskError("simulated transient teardown failure") + + monkeypatch.setattr("engine.ecs.start_range_teardown", fail_teardown) + with pytest.raises(CTFRangeError): + recover_participant_range( + participant.pk, + strategy=RecoveryStrategy.REASSIGN_SPARE.value, + operator=organizer_user, + spare_range_instance_id=spare_range.pk, + ) + + Workspace.objects.filter(pk=event_with_scenario.workspace_id).update(egress_policy="none") + with pytest.raises(CTFRangeError, match="No compatible spare"): + recover_participant_range( + participant.pk, + strategy=RecoveryStrategy.REASSIGN_SPARE.value, + operator=organizer_user, + spare_range_instance_id=spare_range.pk, + ) + + old_range.refresh_from_db() + spare.refresh_from_db() + assert old_range.status == ResourceStatus.READY.value + assert spare.consumed_by_id == participant.pk + assert RangeInstance.objects.get(pk=spare_range.pk).user_id == second_participant_user.pk + @pytest.mark.django_db def test_retry_after_teardown_failure_resumes_without_duplicating( self, monkeypatch, event_with_scenario, rich_participant, second_participant_user, organizer_user @@ -689,82 +758,3 @@ def flaky_teardown(request_id): participant.refresh_from_db() assert participant.range_instance_id == spare_range.pk - - -class TestValidationAndFailures: - @pytest.mark.django_db - def test_participant_not_found(self, organizer_user): - uuid4_2 = uuid4() - with pytest.raises(CTFNotFoundError): - recover_participant_range( - uuid4_2, - strategy=RecoveryStrategy.REBUILD.value, - operator=organizer_user, - ) - - @pytest.mark.django_db - def test_invalid_strategy(self, rich_participant, organizer_user): - participant, _ = rich_participant - with pytest.raises(CTFValidationError): - recover_participant_range( - participant.pk, - strategy="not_a_real_strategy", - operator=organizer_user, - ) - assert not CTFRangeRecovery.objects.filter(participant=participant).exists() - - @pytest.mark.django_db - def test_unregistered_participant_rejected(self, event_with_scenario, organizer_user): - participant = CTFParticipant.objects.create( - event=event_with_scenario, - user=None, - email="unregistered@test.com", - name="Unregistered", - status=ParticipantStatus.REGISTERED.value, - ) - with pytest.raises(CTFValidationError, match="registered"): - recover_participant_range( - participant.pk, - strategy=RecoveryStrategy.REBUILD.value, - operator=organizer_user, - ) - - @pytest.mark.django_db - def test_participant_with_no_range_rejected(self, event_with_scenario, participant_user, organizer_user): - participant = CTFParticipant.objects.create( - event=event_with_scenario, - user=participant_user, - email=participant_user.email, - name="No Range Participant", - status=ParticipantStatus.ACTIVE.value, - registered_at=timezone.now(), - ) - with pytest.raises(CTFRangeError, match="no range"): - recover_participant_range( - participant.pk, - strategy=RecoveryStrategy.REBUILD.value, - operator=organizer_user, - ) - - -class TestGetRecoveryStatus: - @pytest.mark.django_db - def test_returns_none_when_no_recovery_exists(self, rich_participant): - participant, _ = rich_participant - assert get_recovery_status(participant.pk) is None - - @pytest.mark.django_db - def test_returns_latest_recovery_after_completion(self, rich_participant, organizer_user): - participant, _ = rich_participant - - recover_participant_range( - participant.pk, - strategy=RecoveryStrategy.REBUILD.value, - operator=organizer_user, - ) - - status = get_recovery_status(participant.pk) - assert status is not None - assert status["phase"] == RecoveryPhase.COMPLETED.value - assert status["strategy"] == RecoveryStrategy.REBUILD.value - assert status["replacement_range_instance_id"] is not None diff --git a/shifter/shifter_platform/tests/ctf/test_services/test_range_recovery_validation.py b/shifter/shifter_platform/tests/ctf/test_services/test_range_recovery_validation.py new file mode 100644 index 000000000..0947c1dd3 --- /dev/null +++ b/shifter/shifter_platform/tests/ctf/test_services/test_range_recovery_validation.py @@ -0,0 +1,94 @@ +"""Validation and status projection for participant range recovery.""" + +from __future__ import annotations + +from uuid import uuid4 + +import pytest +from django.utils import timezone + +from ctf.enums import ParticipantStatus, RecoveryPhase, RecoveryStrategy +from ctf.exceptions import CTFNotFoundError, CTFRangeError, CTFValidationError +from ctf.models import CTFParticipant, CTFRangeRecovery +from ctf.services.range.recovery import get_recovery_status, recover_participant_range +from tests.ctf.test_services.test_range_recovery import event_with_scenario as event_with_scenario +from tests.ctf.test_services.test_range_recovery import rich_participant as rich_participant +from tests.ctf.test_services.test_range_recovery import team_and_bracket as team_and_bracket + + +class TestValidationAndFailures: + @pytest.mark.django_db + def test_participant_not_found(self, organizer_user): + with pytest.raises(CTFNotFoundError): + recover_participant_range( + uuid4(), + strategy=RecoveryStrategy.REBUILD.value, + operator=organizer_user, + ) + + @pytest.mark.django_db + def test_invalid_strategy(self, rich_participant, organizer_user): + participant, _ = rich_participant + with pytest.raises(CTFValidationError): + recover_participant_range( + participant.pk, + strategy="not_a_real_strategy", + operator=organizer_user, + ) + assert not CTFRangeRecovery.objects.filter(participant=participant).exists() + + @pytest.mark.django_db + def test_unregistered_participant_rejected(self, event_with_scenario, organizer_user): + participant = CTFParticipant.objects.create( + event=event_with_scenario, + user=None, + email="unregistered@test.com", + name="Unregistered", + status=ParticipantStatus.REGISTERED.value, + ) + with pytest.raises(CTFValidationError, match="registered"): + recover_participant_range( + participant.pk, + strategy=RecoveryStrategy.REBUILD.value, + operator=organizer_user, + ) + + @pytest.mark.django_db + def test_participant_with_no_range_rejected(self, event_with_scenario, participant_user, organizer_user): + participant = CTFParticipant.objects.create( + event=event_with_scenario, + user=participant_user, + email=participant_user.email, + name="No Range Participant", + status=ParticipantStatus.ACTIVE.value, + registered_at=timezone.now(), + ) + with pytest.raises(CTFRangeError, match="no range"): + recover_participant_range( + participant.pk, + strategy=RecoveryStrategy.REBUILD.value, + operator=organizer_user, + ) + + +class TestGetRecoveryStatus: + @pytest.mark.django_db + def test_returns_none_when_no_recovery_exists(self, rich_participant): + participant, _ = rich_participant + assert get_recovery_status(participant.pk) is None + + @pytest.mark.django_db + def test_returns_latest_recovery_after_completion(self, rich_participant, organizer_user): + participant, _ = rich_participant + + recover_participant_range( + participant.pk, + strategy=RecoveryStrategy.REBUILD.value, + operator=organizer_user, + ) + + status = get_recovery_status(participant.pk) + assert status is not None + assert status["phase"] == RecoveryPhase.COMPLETED.value + assert status["strategy"] == RecoveryStrategy.REBUILD.value + assert status["replacement_range_instance_id"] is not None diff --git a/shifter/shifter_platform/tests/ctf/test_services/test_range_spares.py b/shifter/shifter_platform/tests/ctf/test_services/test_range_spares.py index 33ed5d8e1..795d24668 100644 --- a/shifter/shifter_platform/tests/ctf/test_services/test_range_spares.py +++ b/shifter/shifter_platform/tests/ctf/test_services/test_range_spares.py @@ -162,6 +162,83 @@ def test_creates_managed_user_owned_rows(self, event_with_scenario, organizer_us assert audit.new_state["event_id"] == str(event_with_scenario.pk) assert audit.new_state["created"] == 2 + @pytest.mark.django_db + def test_spares_pin_event_egress_without_rebinding_owner_workspace(self, event_with_scenario, organizer_user): + from engine.models import Range as EngineRange + from workspaces.models import Workspace + + Workspace.objects.filter(pk=event_with_scenario.workspace_id).update(egress_policy="none") + + result = provision_event_spares(event_with_scenario.pk, 2, operator=organizer_user) + + assert result["created"] == 2 + for spare in CTFSpareRange.objects.filter(event=event_with_scenario): + instance = RangeInstance.objects.get(pk=spare.range_instance_id) + engine_range = EngineRange.objects.get(request__request_id=instance.request.request_id) + assert instance.user_id == spare.owner_user_id + assert instance.workspace_id != event_with_scenario.workspace_id + assert engine_range.workspace_id == instance.workspace_id + assert engine_range.egress_mode == "none" + launch_audit = AuditLog.objects.get( + action=AuditAction.PROVISION, + actor_id=spare.owner_user_id, + new_state__request_id=str(instance.request.request_id), + ) + assert launch_audit.new_state["egress_policy_workspace_id"] == event_with_scenario.workspace_id + assert launch_audit.new_state["egress_mode"] == "none" + + @pytest.mark.django_db + def test_two_participants_use_event_policy_with_distinct_owner_scopes( + self, event_with_scenario, participant_user, second_participant_user + ): + from ctf.services.range.provision import provision_participant_range + from engine.models import Range as EngineRange + from workspaces.models import Workspace + + Workspace.objects.filter(pk=event_with_scenario.workspace_id).update(egress_policy="none") + owner_workspaces = set() + for user in (participant_user, second_participant_user): + participant = CTFParticipant.objects.create( + event=event_with_scenario, + user=user, + email=user.email, + name=user.username, + status=ParticipantStatus.ACTIVE.value, + ) + result = provision_participant_range(participant.pk) + instance = RangeInstance.objects.get(pk=result["range_instance_id"]) + engine_range = EngineRange.objects.get(request__request_id=instance.request.request_id) + assert instance.user_id == user.pk + assert instance.workspace_id != event_with_scenario.workspace_id + assert engine_range.workspace_id == instance.workspace_id + assert engine_range.egress_mode == "none" + owner_workspaces.add(instance.workspace_id) + assert len(owner_workspaces) == 2 + + @pytest.mark.django_db + def test_archived_event_workspace_denies_participant_launch_without_personal_fallback( + self, event_with_scenario, participant_user + ): + from django.utils import timezone + + from ctf.exceptions import CTFRangeError + from ctf.services.range.provision import provision_participant_range + from workspaces.models import Workspace + + Workspace.objects.filter(pk=event_with_scenario.workspace_id).update(archived_at=timezone.now()) + participant = CTFParticipant.objects.create( + event=event_with_scenario, + user=participant_user, + email=participant_user.email, + name="Denied Participant", + status=ParticipantStatus.ACTIVE.value, + ) + + with pytest.raises(CTFRangeError): + provision_participant_range(participant.pk) + + assert not RangeInstance.objects.filter(user_id=participant_user.pk).exists() + @pytest.mark.django_db def test_top_up_is_idempotent_at_the_same_target(self, event_with_scenario, organizer_user): provision_event_spares(event_with_scenario.pk, 2, operator=organizer_user)