feat(eks): run the runtime-plugin sandbox pool on EKS

docs/architecture/external-scenario-runtime-design.md

@@ -16,7 +16,12 @@ policy and independent wheel execution; live cloud isolation is not yet qualifie
 Worker jobs also require the `gvisor` RuntimeClass and dedicated plugin nodes.
 Admission rejects default runtimes and platform-node placement. The GCP deployment
 declares a bounded GKE Sandbox pool with warm capacity; AWS declares a dedicated
-EKS sandbox node pool with the same workload restrictions. An installation without the required runtime fails closed.
+EKS sandbox node pool with the same workload restrictions. Pool membership is the
+`node-restriction.kubernetes.io/shifter-pool=runtime-plugin` label, which a
+kubelet cannot set. GKE applies it through its control plane; on EKS a trusted
+labeler applies it only to running nodes that AWS has tagged as members of the
+pool's own Auto Scaling group, and admission confines that labeler to the one
+label (#2526). An installation without the required runtime fails closed.
 It cannot run tenant code in the default container runtime as a fallback.
 
 Tenant admins can now assign installed versions to registered packs and select

platform/charts/shifter/templates/runtime-plugin-node-labeler.yaml

@@ -0,0 +1,164 @@
+{{- if and (eq .Values.provider.name "aws") .Values.capabilities.kubernetesJobLauncher .Values.runtimePluginPool.labeler.enabled }}
+{{- $labeler := .Values.runtimePluginPool.labeler }}
+{{- $subject := printf "system:serviceaccount:%s:node-pool-labeler" .Values.namespaces.platform }}
+# Trusted runtime-plugin node-pool labeler (#2526). NodeRestriction stops a
+# kubelet from setting node-restriction.kubernetes.io/shifter-pool, and EKS
+# managed node groups can only apply labels through the kubelet, so this
+# controller applies the pool label to nodes in the runtime-plugin node group's
+# Auto Scaling group (shared.cloud.eks_node_pool_labeler).
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: node-pool-labeler
+  namespace: {{ .Values.namespaces.platform }}
+  labels:
+    {{- include "shifter.partOfLabels" . | nindent 4 }}
+  {{- with .Values.identity.serviceAccountRoleArns.nodePoolLabeler }}
+  annotations:
+    eks.amazonaws.com/role-arn: {{ . | quote }}
+  {{- end }}
+automountServiceAccountToken: false
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: shifter-node-pool-labeler
+  labels:
+    {{- include "shifter.partOfLabels" . | nindent 4 }}
+rules:
+  - apiGroups: [""]
+    resources: [nodes]
+    verbs: [get, list, watch, patch]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: shifter-node-pool-labeler
+  labels:
+    {{- include "shifter.partOfLabels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: shifter-node-pool-labeler
+subjects:
+  - kind: ServiceAccount
+    name: node-pool-labeler
+    namespace: {{ .Values.namespaces.platform }}
+---
+# The labeler may toggle exactly one label on a Node and nothing else.
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicy
+metadata:
+  name: restrict-node-pool-labeler
+spec:
+  failurePolicy: Fail
+  matchConstraints:
+    resourceRules:
+      - apiGroups: [""]
+        apiVersions: [v1]
+        operations: [UPDATE]
+        resources: [nodes]
+  matchConditions:
+    - name: labeler-identity
+      expression: "request.userInfo.username == '{{ $subject }}'"
+  variables:
+    - name: key
+      expression: "'node-restriction.kubernetes.io/shifter-pool'"
+    - name: newLabels
+      expression: "has(object.metadata.labels) ? object.metadata.labels : {}"
+    - name: oldLabels
+      expression: "has(oldObject.metadata.labels) ? oldObject.metadata.labels : {}"
+  validations:
+    - expression: "object.spec == oldObject.spec"
+      message: The node-pool labeler may not change a node's spec.
+    - expression: >-
+        (has(object.metadata.annotations) ? object.metadata.annotations : {}) ==
+        (has(oldObject.metadata.annotations) ? oldObject.metadata.annotations : {})
+      message: The node-pool labeler may not change node annotations.
+    - expression: >-
+        variables.newLabels.all(k, k == variables.key || (k in variables.oldLabels && variables.oldLabels[k] == variables.newLabels[k])) &&
+        variables.oldLabels.all(k, k == variables.key || k in variables.newLabels)
+      message: The node-pool labeler may change only the runtime-plugin pool label.
+    - expression: "!(variables.key in variables.newLabels) || variables.newLabels[variables.key] == 'runtime-plugin'"
+      message: The node-pool labeler may only assign the runtime-plugin pool.
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicyBinding
+metadata:
+  name: restrict-node-pool-labeler
+spec:
+  policyName: restrict-node-pool-labeler
+  validationActions: [Deny]
+{{- if not .Values.network.kubernetesApiCidrs }}
+{{- fail "network.kubernetesApiCidrs is required for the node-pool labeler" }}
+{{- end }}
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-node-pool-labeler-kubernetes-api-egress
+  namespace: {{ .Values.namespaces.platform }}
+  labels:
+    {{- include "shifter.partOfLabels" . | nindent 4 }}
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/component: node-pool-labeler
+  policyTypes:
+    - Egress
+  egress:
+    - to:
+        {{- range .Values.network.kubernetesApiCidrs }}
+        - ipBlock:
+            cidr: {{ . | quote }}
+        {{- end }}
+      ports:
+        - protocol: TCP
+          port: 443
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: node-pool-labeler
+  namespace: {{ .Values.namespaces.platform }}
+  labels:
+    {{- include "shifter.partOfLabels" . | nindent 4 }}
+    app.kubernetes.io/component: node-pool-labeler
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/part-of: shifter
+      app.kubernetes.io/component: node-pool-labeler
+  template:
+    metadata:
+      labels:
+        {{- include "shifter.partOfLabels" . | nindent 8 }}
+        app.kubernetes.io/component: node-pool-labeler
+    spec:
+      {{- include "shifter.podSecurityContext" . | nindent 6 }}
+      serviceAccountName: node-pool-labeler
+      automountServiceAccountToken: true
+      containers:
+        - name: node-pool-labeler
+          image: {{ include "shifter.portalImage" . }}
+          imagePullPolicy: {{ .Values.imagePullPolicy }}
+          {{- include "shifter.containerSecurityContextApp" . | nindent 10 }}
+          {{- include "shifter.tmpVolumeMount" . | nindent 10 }}
+          command: ["python", "-m", "shared.cloud.eks_node_pool_labeler"]
+          env:
+            - name: RUNTIME_PLUGIN_NODE_GROUP_ASG
+              value: {{ required "runtimePluginPool.labeler.nodeGroupAsg is required" $labeler.nodeGroupAsg | quote }}
+            - name: AWS_REGION
+              value: {{ required "runtimeEnv.AWS_REGION is required for the node-pool labeler" .Values.runtimeEnv.AWS_REGION | quote }}
+          livenessProbe:
+            exec:
+              command: ["sh", "-c", "find /tmp/node-pool-labeler-heartbeat -mmin -2 | grep -q ."]
+            initialDelaySeconds: 60
+            periodSeconds: 30
+          resources:
+            {{- toYaml $labeler.resources | nindent 12 }}
+      {{- include "shifter.tmpVolume" . | nindent 6 }}
+{{- end }}

platform/charts/shifter/tests/test_chart_contract.py

@@ -123,6 +123,39 @@ def test_aws_supplies_gvisor_runtime_class_for_only_the_isolated_pool(self) -> N
         _, gcp = _render(VALUES_FILES["gcp-dev"])
         self.assertFalse(any(doc["kind"] == "RuntimeClass" for doc in gcp))
 
+    def test_aws_node_pool_labeler_is_bounded_to_the_pool_label(self) -> None:
+        """#2526: the trusted labeler renders only when enabled and may toggle only the pool label."""
+        _, default = _render(VALUES_FILES["aws-dev"])
+        self.assertFalse(any(_identity(doc)[1] == "node-pool-labeler" for doc in default))
+        node_group = "eks-runtime-plugins-1234-abcd"
+        labeler_role = "arn:aws:iam::123456789012:role/shifter-dev-node-pool-labeler"
+        rendered = _helm(
+            "template", "contract-test", str(CHART_DIR), "-f", str(VALUES_FILES["aws-dev"]),
+            "--set", "runtimePluginPool.labeler.enabled=true",
+            "--set", f"runtimePluginPool.labeler.nodeGroupAsg={node_group}",
+            "--set", f"identity.serviceAccountRoleArns.nodePoolLabeler={labeler_role}",
+            "--set", "runtimeEnv.AWS_REGION=us-east-2",
+            "--set", "network.kubernetesApiCidrs={10.100.0.0/16}",
+        ).stdout
+        docs = {_identity(doc): doc for doc in yaml.safe_load_all(rendered) if isinstance(doc, dict)}
+        account = docs[("ServiceAccount", "node-pool-labeler")]
+        self.assertEqual(account["metadata"]["annotations"], {"eks.amazonaws.com/role-arn": labeler_role})
+        self.assertEqual(docs[("ClusterRole", "shifter-node-pool-labeler")]["rules"], [
+            {"apiGroups": [""], "resources": ["nodes"], "verbs": ["get", "list", "watch", "patch"]},
+        ])
+        policy = docs[("ValidatingAdmissionPolicy", "restrict-node-pool-labeler")]
+        self.assertEqual(policy["spec"]["matchConditions"][0]["expression"],
+                         "request.userInfo.username == 'system:serviceaccount:shifter-platform:node-pool-labeler'")
+        self.assertEqual(docs[("ValidatingAdmissionPolicyBinding", "restrict-node-pool-labeler")]["spec"][
+            "validationActions"], ["Deny"])
+        container = docs[("Deployment", "node-pool-labeler")]["spec"]["template"]["spec"]["containers"][0]
+        self.assertEqual(container["command"], ["python", "-m", "shared.cloud.eks_node_pool_labeler"])
+        self.assertIn({"name": "RUNTIME_PLUGIN_NODE_GROUP_ASG", "value": node_group}, container["env"])
+        self.assertIn(("NetworkPolicy", "allow-node-pool-labeler-kubernetes-api-egress"), docs)
+        # GKE applies the pool label itself; the labeler never renders there.
+        _, gcp = _render(VALUES_FILES["gcp-dev"])
+        self.assertFalse(any(_identity(doc)[1] == "node-pool-labeler" for doc in gcp))
+
     def test_chart_has_schema_and_all_backend_profiles(self) -> None:
         schema = json.loads((CHART_DIR / "values.schema.json").read_text())
         self.assertEqual(schema["$schema"], "https://json-schema.org/draft/2020-12/schema")

platform/charts/shifter/values.schema.json

@@ -226,6 +226,23 @@
         "egressExcept": { "$ref": "#/$defs/cidrList" }
       }
     },
+    "runtimePluginPool": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["labeler"],
+      "properties": {
+        "labeler": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["enabled", "nodeGroupAsg", "resources"],
+          "properties": {
+            "enabled": { "type": "boolean" },
+            "nodeGroupAsg": { "type": "string", "pattern": "^[A-Za-z0-9_+=,.@/:-]{0,255}$" },
+            "resources": { "type": "object" }
+          }
+        }
+      }
+    },
     "serviceAccounts": {
       "type": "object",
       "additionalProperties": false,

platform/charts/shifter/values.yaml

@@ -39,6 +39,23 @@ featureArtifactAcquisition:
   egressCidrs: []
   egressExcept: []
 
+# Trusted runtime-plugin node-pool labeler (AWS EKS, #2526). Applies the
+# node-restriction.kubernetes.io/shifter-pool=runtime-plugin label to nodes in the
+# nodeGroupAsg Auto Scaling group. The AWS renderer enables it with the pool.
+runtimePluginPool:
+  labeler:
+    enabled: false
+    nodeGroupAsg: ""
+    # Measured: the platform package, boto3 and the Kubernetes client take about
+    # 180 MiB resident before the first reconcile pass.
+    resources:
+      requests:
+        cpu: 10m
+        memory: 192Mi
+      limits:
+        cpu: 200m
+        memory: 320Mi
+
 namespaces:
   platform: shifter-platform
   jobs: shifter-jobs

platform/terraform/environments/dev/eks/main.tf

@@ -50,6 +50,9 @@ module "eks" {
   oidc_thumbprints         = var.oidc_thumbprints
   secret_names             = local.secret_names
   storage_bucket_name      = var.runtime_env["STORAGE_BUCKET_NAME"]
+  # Exclusive gVisor pool for tenant runtime-plugin workers (#2526); the
+  # nodePoolLabeler identity below applies its trusted pool label.
+  enable_runtime_plugins = true
   workload_identities = {
     cni = {
       namespace       = "kube-system"
@@ -126,6 +129,15 @@ module "eks" {
       secret_names                 = []
       feature_artifact_store_write = true
     }
+    # Trusted runtime-plugin node-pool labeler (#2526): labels only nodes backed
+    # by the plugin pool's node role. No secrets, no database.
+    nodePoolLabeler = {
+      namespace         = "shifter-platform"
+      service_account   = "node-pool-labeler"
+      policy_arns       = []
+      secret_names      = []
+      node_pool_labeler = true
+    }
     # One-shot schema-migration + content-bootstrap Job (AWS EKS parity with the GCP
     # platform-migrate Job, #1826). aws_eks.py runs it once before the chart install so
     # the workloads deploy with SKIP_MIGRATIONS=1 and never race per-pod migrations on

platform/terraform/environments/dev/eks/outputs.tf

@@ -90,3 +90,8 @@ output "model_broker" {
   description = "Applied private model broker configuration."
   value       = module.eks.model_broker
 }
+
+output "runtime_plugin_node_group_asg" {
+  description = "Auto Scaling group of the runtime-plugin pool, consumed by the trusted node-pool labeler (#2526)."
+  value       = module.eks.runtime_plugin_node_group_asg
+}

platform/terraform/modules/portal/eks/iam.tf

@@ -33,6 +33,11 @@ locals {
     identity_name
     if identity.feature_artifact_store_read
   ])
+  workload_node_pool_labeler = toset([
+    for identity_name, identity in var.workload_identities :
+    identity_name
+    if identity.node_pool_labeler
+  ])
   workload_platform_application_access = toset([
     for identity_name, identity in var.workload_identities :
     identity_name
@@ -494,6 +499,27 @@ resource "aws_iam_role_policy" "workload_feature_artifact_store" {
   })
 }
 
+# Trusted runtime-plugin node-pool labeler (#2526): reads each EKS node's
+# instance (state, DNS name and its AWS-set aws:autoscaling:groupName tag).
+# ec2:DescribeInstances has no resource-level scoping.
+resource "aws_iam_role_policy" "workload_node_pool_labeler" {
+  for_each = local.workload_node_pool_labeler
+
+  name = "node-pool-labeler"
+  role = aws_iam_role.workload[each.key].id
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Sid      = "DescribeNodeInstances"
+        Effect   = "Allow"
+        Action   = ["ec2:DescribeInstances"]
+        Resource = "*"
+      },
+    ]
+  })
+}
+
 # Deploy content bootstrap (acquire_feature_artifacts, ADR-034-R12): verifies an
 # inventoried artifact still exists before reusing it. HEAD needs s3:GetObject,
 # and only ListBucket turns a missing key into 404 instead of 403. With no

platform/terraform/modules/portal/eks/outputs.tf

@@ -100,3 +100,8 @@ output "ingress_waf_acl_arn" {
   description = "Regional WAF ACL ARN for the ingress load balancer."
   value       = aws_wafv2_web_acl.ingress.arn
 }
+
+output "runtime_plugin_node_group_asg" {
+  description = "Auto Scaling group of the runtime-plugin pool; the trusted labeler labels only its nodes. Empty when the pool is disabled."
+  value       = var.enable_runtime_plugins ? aws_eks_node_group.runtime_plugins[0].resources[0].autoscaling_groups[0].name : ""
+}