diff --git a/docs/architecture/external-scenario-runtime-design.md b/docs/architecture/external-scenario-runtime-design.md index 1ac4ea08a..7210c4050 100644 --- a/docs/architecture/external-scenario-runtime-design.md +++ b/docs/architecture/external-scenario-runtime-design.md @@ -16,7 +16,12 @@ policy and independent wheel execution; live cloud isolation is not yet qualifie Worker jobs also require the `gvisor` RuntimeClass and dedicated plugin nodes. Admission rejects default runtimes and platform-node placement. The GCP deployment declares a bounded GKE Sandbox pool with warm capacity; AWS declares a dedicated -EKS sandbox node pool with the same workload restrictions. An installation without the required runtime fails closed. +EKS sandbox node pool with the same workload restrictions. Pool membership is the +`node-restriction.kubernetes.io/shifter-pool=runtime-plugin` label, which a +kubelet cannot set. GKE applies it through its control plane; on EKS a trusted +labeler applies it only to running nodes that AWS has tagged as members of the +pool's own Auto Scaling group, and admission confines that labeler to the one +label (#2526). An installation without the required runtime fails closed. It cannot run tenant code in the default container runtime as a fallback. Tenant admins can now assign installed versions to registered packs and select diff --git a/platform/charts/shifter/templates/runtime-plugin-node-labeler.yaml b/platform/charts/shifter/templates/runtime-plugin-node-labeler.yaml new file mode 100644 index 000000000..45c2524d6 --- /dev/null +++ b/platform/charts/shifter/templates/runtime-plugin-node-labeler.yaml @@ -0,0 +1,164 @@ +{{- if and (eq .Values.provider.name "aws") .Values.capabilities.kubernetesJobLauncher .Values.runtimePluginPool.labeler.enabled }} +{{- $labeler := .Values.runtimePluginPool.labeler }} +{{- $subject := printf "system:serviceaccount:%s:node-pool-labeler" .Values.namespaces.platform }} +# Trusted runtime-plugin node-pool labeler (#2526). NodeRestriction stops a +# kubelet from setting node-restriction.kubernetes.io/shifter-pool, and EKS +# managed node groups can only apply labels through the kubelet, so this +# controller applies the pool label to nodes in the runtime-plugin node group's +# Auto Scaling group (shared.cloud.eks_node_pool_labeler). +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-pool-labeler + namespace: {{ .Values.namespaces.platform }} + labels: + {{- include "shifter.partOfLabels" . | nindent 4 }} + {{- with .Values.identity.serviceAccountRoleArns.nodePoolLabeler }} + annotations: + eks.amazonaws.com/role-arn: {{ . | quote }} + {{- end }} +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: shifter-node-pool-labeler + labels: + {{- include "shifter.partOfLabels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: [nodes] + verbs: [get, list, watch, patch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: shifter-node-pool-labeler + labels: + {{- include "shifter.partOfLabels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: shifter-node-pool-labeler +subjects: + - kind: ServiceAccount + name: node-pool-labeler + namespace: {{ .Values.namespaces.platform }} +--- +# The labeler may toggle exactly one label on a Node and nothing else. +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: restrict-node-pool-labeler +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: [v1] + operations: [UPDATE] + resources: [nodes] + matchConditions: + - name: labeler-identity + expression: "request.userInfo.username == '{{ $subject }}'" + variables: + - name: key + expression: "'node-restriction.kubernetes.io/shifter-pool'" + - name: newLabels + expression: "has(object.metadata.labels) ? object.metadata.labels : {}" + - name: oldLabels + expression: "has(oldObject.metadata.labels) ? oldObject.metadata.labels : {}" + validations: + - expression: "object.spec == oldObject.spec" + message: The node-pool labeler may not change a node's spec. + - expression: >- + (has(object.metadata.annotations) ? object.metadata.annotations : {}) == + (has(oldObject.metadata.annotations) ? oldObject.metadata.annotations : {}) + message: The node-pool labeler may not change node annotations. + - expression: >- + variables.newLabels.all(k, k == variables.key || (k in variables.oldLabels && variables.oldLabels[k] == variables.newLabels[k])) && + variables.oldLabels.all(k, k == variables.key || k in variables.newLabels) + message: The node-pool labeler may change only the runtime-plugin pool label. + - expression: "!(variables.key in variables.newLabels) || variables.newLabels[variables.key] == 'runtime-plugin'" + message: The node-pool labeler may only assign the runtime-plugin pool. +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: restrict-node-pool-labeler +spec: + policyName: restrict-node-pool-labeler + validationActions: [Deny] +{{- if not .Values.network.kubernetesApiCidrs }} +{{- fail "network.kubernetesApiCidrs is required for the node-pool labeler" }} +{{- end }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-node-pool-labeler-kubernetes-api-egress + namespace: {{ .Values.namespaces.platform }} + labels: + {{- include "shifter.partOfLabels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + app.kubernetes.io/component: node-pool-labeler + policyTypes: + - Egress + egress: + - to: + {{- range .Values.network.kubernetesApiCidrs }} + - ipBlock: + cidr: {{ . | quote }} + {{- end }} + ports: + - protocol: TCP + port: 443 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: node-pool-labeler + namespace: {{ .Values.namespaces.platform }} + labels: + {{- include "shifter.partOfLabels" . | nindent 4 }} + app.kubernetes.io/component: node-pool-labeler +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/part-of: shifter + app.kubernetes.io/component: node-pool-labeler + template: + metadata: + labels: + {{- include "shifter.partOfLabels" . | nindent 8 }} + app.kubernetes.io/component: node-pool-labeler + spec: + {{- include "shifter.podSecurityContext" . | nindent 6 }} + serviceAccountName: node-pool-labeler + automountServiceAccountToken: true + containers: + - name: node-pool-labeler + image: {{ include "shifter.portalImage" . }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + {{- include "shifter.containerSecurityContextApp" . | nindent 10 }} + {{- include "shifter.tmpVolumeMount" . | nindent 10 }} + command: ["python", "-m", "shared.cloud.eks_node_pool_labeler"] + env: + - name: RUNTIME_PLUGIN_NODE_GROUP_ASG + value: {{ required "runtimePluginPool.labeler.nodeGroupAsg is required" $labeler.nodeGroupAsg | quote }} + - name: AWS_REGION + value: {{ required "runtimeEnv.AWS_REGION is required for the node-pool labeler" .Values.runtimeEnv.AWS_REGION | quote }} + livenessProbe: + exec: + command: ["sh", "-c", "find /tmp/node-pool-labeler-heartbeat -mmin -2 | grep -q ."] + initialDelaySeconds: 60 + periodSeconds: 30 + resources: + {{- toYaml $labeler.resources | nindent 12 }} + {{- include "shifter.tmpVolume" . | nindent 6 }} +{{- end }} diff --git a/platform/charts/shifter/tests/test_chart_contract.py b/platform/charts/shifter/tests/test_chart_contract.py index 1e235dab9..51d831f97 100644 --- a/platform/charts/shifter/tests/test_chart_contract.py +++ b/platform/charts/shifter/tests/test_chart_contract.py @@ -123,6 +123,39 @@ def test_aws_supplies_gvisor_runtime_class_for_only_the_isolated_pool(self) -> N _, gcp = _render(VALUES_FILES["gcp-dev"]) self.assertFalse(any(doc["kind"] == "RuntimeClass" for doc in gcp)) + def test_aws_node_pool_labeler_is_bounded_to_the_pool_label(self) -> None: + """#2526: the trusted labeler renders only when enabled and may toggle only the pool label.""" + _, default = _render(VALUES_FILES["aws-dev"]) + self.assertFalse(any(_identity(doc)[1] == "node-pool-labeler" for doc in default)) + node_group = "eks-runtime-plugins-1234-abcd" + labeler_role = "arn:aws:iam::123456789012:role/shifter-dev-node-pool-labeler" + rendered = _helm( + "template", "contract-test", str(CHART_DIR), "-f", str(VALUES_FILES["aws-dev"]), + "--set", "runtimePluginPool.labeler.enabled=true", + "--set", f"runtimePluginPool.labeler.nodeGroupAsg={node_group}", + "--set", f"identity.serviceAccountRoleArns.nodePoolLabeler={labeler_role}", + "--set", "runtimeEnv.AWS_REGION=us-east-2", + "--set", "network.kubernetesApiCidrs={10.100.0.0/16}", + ).stdout + docs = {_identity(doc): doc for doc in yaml.safe_load_all(rendered) if isinstance(doc, dict)} + account = docs[("ServiceAccount", "node-pool-labeler")] + self.assertEqual(account["metadata"]["annotations"], {"eks.amazonaws.com/role-arn": labeler_role}) + self.assertEqual(docs[("ClusterRole", "shifter-node-pool-labeler")]["rules"], [ + {"apiGroups": [""], "resources": ["nodes"], "verbs": ["get", "list", "watch", "patch"]}, + ]) + policy = docs[("ValidatingAdmissionPolicy", "restrict-node-pool-labeler")] + self.assertEqual(policy["spec"]["matchConditions"][0]["expression"], + "request.userInfo.username == 'system:serviceaccount:shifter-platform:node-pool-labeler'") + self.assertEqual(docs[("ValidatingAdmissionPolicyBinding", "restrict-node-pool-labeler")]["spec"][ + "validationActions"], ["Deny"]) + container = docs[("Deployment", "node-pool-labeler")]["spec"]["template"]["spec"]["containers"][0] + self.assertEqual(container["command"], ["python", "-m", "shared.cloud.eks_node_pool_labeler"]) + self.assertIn({"name": "RUNTIME_PLUGIN_NODE_GROUP_ASG", "value": node_group}, container["env"]) + self.assertIn(("NetworkPolicy", "allow-node-pool-labeler-kubernetes-api-egress"), docs) + # GKE applies the pool label itself; the labeler never renders there. + _, gcp = _render(VALUES_FILES["gcp-dev"]) + self.assertFalse(any(_identity(doc)[1] == "node-pool-labeler" for doc in gcp)) + def test_chart_has_schema_and_all_backend_profiles(self) -> None: schema = json.loads((CHART_DIR / "values.schema.json").read_text()) self.assertEqual(schema["$schema"], "https://json-schema.org/draft/2020-12/schema") diff --git a/platform/charts/shifter/values.schema.json b/platform/charts/shifter/values.schema.json index 20ac1ef35..638309de5 100644 --- a/platform/charts/shifter/values.schema.json +++ b/platform/charts/shifter/values.schema.json @@ -226,6 +226,23 @@ "egressExcept": { "$ref": "#/$defs/cidrList" } } }, + "runtimePluginPool": { + "type": "object", + "additionalProperties": false, + "required": ["labeler"], + "properties": { + "labeler": { + "type": "object", + "additionalProperties": false, + "required": ["enabled", "nodeGroupAsg", "resources"], + "properties": { + "enabled": { "type": "boolean" }, + "nodeGroupAsg": { "type": "string", "pattern": "^[A-Za-z0-9_+=,.@/:-]{0,255}$" }, + "resources": { "type": "object" } + } + } + } + }, "serviceAccounts": { "type": "object", "additionalProperties": false, diff --git a/platform/charts/shifter/values.yaml b/platform/charts/shifter/values.yaml index 9f15e9b66..c4597dbce 100644 --- a/platform/charts/shifter/values.yaml +++ b/platform/charts/shifter/values.yaml @@ -39,6 +39,23 @@ featureArtifactAcquisition: egressCidrs: [] egressExcept: [] +# Trusted runtime-plugin node-pool labeler (AWS EKS, #2526). Applies the +# node-restriction.kubernetes.io/shifter-pool=runtime-plugin label to nodes in the +# nodeGroupAsg Auto Scaling group. The AWS renderer enables it with the pool. +runtimePluginPool: + labeler: + enabled: false + nodeGroupAsg: "" + # Measured: the platform package, boto3 and the Kubernetes client take about + # 180 MiB resident before the first reconcile pass. + resources: + requests: + cpu: 10m + memory: 192Mi + limits: + cpu: 200m + memory: 320Mi + namespaces: platform: shifter-platform jobs: shifter-jobs diff --git a/platform/terraform/environments/dev/eks/main.tf b/platform/terraform/environments/dev/eks/main.tf index 7faa9e765..e072772c8 100644 --- a/platform/terraform/environments/dev/eks/main.tf +++ b/platform/terraform/environments/dev/eks/main.tf @@ -50,6 +50,9 @@ module "eks" { oidc_thumbprints = var.oidc_thumbprints secret_names = local.secret_names storage_bucket_name = var.runtime_env["STORAGE_BUCKET_NAME"] + # Exclusive gVisor pool for tenant runtime-plugin workers (#2526); the + # nodePoolLabeler identity below applies its trusted pool label. + enable_runtime_plugins = true workload_identities = { cni = { namespace = "kube-system" @@ -126,6 +129,15 @@ module "eks" { secret_names = [] feature_artifact_store_write = true } + # Trusted runtime-plugin node-pool labeler (#2526): labels only nodes backed + # by the plugin pool's node role. No secrets, no database. + nodePoolLabeler = { + namespace = "shifter-platform" + service_account = "node-pool-labeler" + policy_arns = [] + secret_names = [] + node_pool_labeler = true + } # One-shot schema-migration + content-bootstrap Job (AWS EKS parity with the GCP # platform-migrate Job, #1826). aws_eks.py runs it once before the chart install so # the workloads deploy with SKIP_MIGRATIONS=1 and never race per-pod migrations on diff --git a/platform/terraform/environments/dev/eks/outputs.tf b/platform/terraform/environments/dev/eks/outputs.tf index c5d3ee524..95c6c246a 100644 --- a/platform/terraform/environments/dev/eks/outputs.tf +++ b/platform/terraform/environments/dev/eks/outputs.tf @@ -90,3 +90,8 @@ output "model_broker" { description = "Applied private model broker configuration." value = module.eks.model_broker } + +output "runtime_plugin_node_group_asg" { + description = "Auto Scaling group of the runtime-plugin pool, consumed by the trusted node-pool labeler (#2526)." + value = module.eks.runtime_plugin_node_group_asg +} diff --git a/platform/terraform/modules/portal/eks/iam.tf b/platform/terraform/modules/portal/eks/iam.tf index 8f50dfa36..19e5da4a5 100644 --- a/platform/terraform/modules/portal/eks/iam.tf +++ b/platform/terraform/modules/portal/eks/iam.tf @@ -33,6 +33,11 @@ locals { identity_name if identity.feature_artifact_store_read ]) + workload_node_pool_labeler = toset([ + for identity_name, identity in var.workload_identities : + identity_name + if identity.node_pool_labeler + ]) workload_platform_application_access = toset([ for identity_name, identity in var.workload_identities : identity_name @@ -494,6 +499,27 @@ resource "aws_iam_role_policy" "workload_feature_artifact_store" { }) } +# Trusted runtime-plugin node-pool labeler (#2526): reads each EKS node's +# instance (state, DNS name and its AWS-set aws:autoscaling:groupName tag). +# ec2:DescribeInstances has no resource-level scoping. +resource "aws_iam_role_policy" "workload_node_pool_labeler" { + for_each = local.workload_node_pool_labeler + + name = "node-pool-labeler" + role = aws_iam_role.workload[each.key].id + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "DescribeNodeInstances" + Effect = "Allow" + Action = ["ec2:DescribeInstances"] + Resource = "*" + }, + ] + }) +} + # Deploy content bootstrap (acquire_feature_artifacts, ADR-034-R12): verifies an # inventoried artifact still exists before reusing it. HEAD needs s3:GetObject, # and only ListBucket turns a missing key into 404 instead of 403. With no diff --git a/platform/terraform/modules/portal/eks/outputs.tf b/platform/terraform/modules/portal/eks/outputs.tf index 48afaa333..b35c92f9c 100644 --- a/platform/terraform/modules/portal/eks/outputs.tf +++ b/platform/terraform/modules/portal/eks/outputs.tf @@ -100,3 +100,8 @@ output "ingress_waf_acl_arn" { description = "Regional WAF ACL ARN for the ingress load balancer." value = aws_wafv2_web_acl.ingress.arn } + +output "runtime_plugin_node_group_asg" { + description = "Auto Scaling group of the runtime-plugin pool; the trusted labeler labels only its nodes. Empty when the pool is disabled." + value = var.enable_runtime_plugins ? aws_eks_node_group.runtime_plugins[0].resources[0].autoscaling_groups[0].name : "" +} diff --git a/platform/terraform/modules/portal/eks/runtime_plugins.tf b/platform/terraform/modules/portal/eks/runtime_plugins.tf index bc266ccf6..bed68c7b6 100644 --- a/platform/terraform/modules/portal/eks/runtime_plugins.tf +++ b/platform/terraform/modules/portal/eks/runtime_plugins.tf @@ -65,17 +65,50 @@ resource "aws_launch_template" "runtime_plugins" { tags = var.tags } +# The pool's own node role keeps its nodes' AWS identity apart from the platform +# pool. Pool membership is the node group's Auto Scaling group: the trusted +# node-pool labeler (shared.cloud.eks_node_pool_labeler) applies the +# node-restriction.kubernetes.io/shifter-pool=runtime-plugin label only to nodes +# whose aws:autoscaling:groupName tag names it. A kubelet cannot set that label +# (NodeRestriction), and only AWS can set aws:-prefixed tags. +resource "aws_iam_role" "runtime_plugin_node" { + count = var.enable_runtime_plugins ? 1 : 0 + name = "${var.cluster_name}-runtime-plugin-node" + permissions_boundary = var.permissions_boundary_arn + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "ec2.amazonaws.com" } + Action = "sts:AssumeRole" + }] + }) + + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "runtime_plugin_node" { + for_each = var.enable_runtime_plugins ? toset([ + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + ]) : toset([]) + + role = aws_iam_role.runtime_plugin_node[0].name + policy_arn = each.value +} + resource "aws_eks_node_group" "runtime_plugins" { count = var.enable_runtime_plugins ? 1 : 0 cluster_name = aws_eks_cluster.this.name node_group_name = "runtime-plugins" - node_role_arn = aws_iam_role.node.arn + node_role_arn = aws_iam_role.runtime_plugin_node[0].arn subnet_ids = [for subnet in aws_subnet.private : subnet.id] version = var.kubernetes_version ami_type = "AL2023_x86_64_STANDARD" instance_types = ["m7i.large"] capacity_type = "ON_DEMAND" - labels = { "node-restriction.kubernetes.io/shifter-pool" = "runtime-plugin" } + # No labels: the pool label is applied by the trusted labeler, never the kubelet. taint { key = "shifter.dev/runtime-plugin" value = "true" @@ -97,7 +130,7 @@ resource "aws_eks_node_group" "runtime_plugins" { # Like the platform node group, these nodes cannot reach Ready until the vpc-cni # and kube-proxy addons exist, so order them after both addons. depends_on = [ - aws_iam_role_policy_attachment.node, + aws_iam_role_policy_attachment.runtime_plugin_node, aws_eks_addon.vpc_cni, aws_eks_addon.kube_proxy, ] diff --git a/platform/terraform/modules/portal/eks/tests/security.tftest.hcl b/platform/terraform/modules/portal/eks/tests/security.tftest.hcl index 052bee73b..edc8de090 100644 --- a/platform/terraform/modules/portal/eks/tests/security.tftest.hcl +++ b/platform/terraform/modules/portal/eks/tests/security.tftest.hcl @@ -266,6 +266,11 @@ variables { service_account = "artifact-acquirer" feature_artifact_store_write = true } + nodePoolLabeler = { + namespace = "shifter-platform" + service_account = "node-pool-labeler" + node_pool_labeler = true + } migrator = { namespace = "shifter-platform" service_account = "migrator" @@ -299,9 +304,8 @@ variables { "guacamole-db", "guacamole-json-auth", ] - # Validate the runtime-plugin sandbox pool contract even though it defaults off - # on EKS (see variables.tf: needs a trusted node-labeler for the restricted - # pool label before it can join). + # Validate the runtime-plugin sandbox pool contract; the trusted node-pool + # labeler applies its restricted pool label (#2526). enable_runtime_plugins = true tags = { Environment = "test" @@ -315,12 +319,26 @@ run "security_contract" { assert { condition = ( aws_eks_node_group.runtime_plugins[0].ami_type == "AL2023_x86_64_STANDARD" && - aws_eks_node_group.runtime_plugins[0].labels["node-restriction.kubernetes.io/shifter-pool"] == "runtime-plugin" && + length(coalesce(aws_eks_node_group.runtime_plugins[0].labels, {})) == 0 && + aws_eks_node_group.runtime_plugins[0].node_role_arn == aws_iam_role.runtime_plugin_node[0].arn && + aws_iam_role.runtime_plugin_node[0].arn != aws_iam_role.node.arn && one(aws_eks_node_group.runtime_plugins[0].taint).key == "shifter.dev/runtime-plugin" && one(aws_eks_node_group.runtime_plugins[0].taint).effect == "NO_SCHEDULE" && aws_eks_node_group.runtime_plugins[0].scaling_config[0].min_size == 1 ) - error_message = "Tenant runtime plugins require a warm, exclusive AL2023 sandbox pool." + error_message = "Tenant runtime plugins require a warm, exclusive AL2023 sandbox pool with its own node role and no kubelet-set labels." + } + + assert { + condition = ( + keys(aws_iam_role_policy.workload_node_pool_labeler) == ["nodePoolLabeler"] && + jsondecode(aws_iam_role_policy.workload_node_pool_labeler["nodePoolLabeler"].policy).Statement[0].Action == ["ec2:DescribeInstances"] && + length(jsondecode(aws_iam_role_policy.workload_node_pool_labeler["nodePoolLabeler"].policy).Statement) == 1 && + !contains(keys(aws_iam_role_policy.workload_platform_application), "nodePoolLabeler") && + !contains(keys(aws_iam_role_policy.workload_rds_iam), "nodePoolLabeler") && + output.runtime_plugin_node_group_asg == "eks-shifter-test-plugins-asg" + ) + error_message = "The node-pool labeler may only read node instances." } assert { diff --git a/platform/terraform/modules/portal/eks/variables.tf b/platform/terraform/modules/portal/eks/variables.tf index 6c0430e7e..bbbe20aaf 100644 --- a/platform/terraform/modules/portal/eks/variables.tf +++ b/platform/terraform/modules/portal/eks/variables.tf @@ -189,6 +189,9 @@ variable "workload_identities" { # missing key answers 404; no KMS, so object bytes stay undecryptable. # Requires storage_bucket_name. feature_artifact_store_read = optional(bool, false) + # The trusted runtime-plugin node-pool labeler (#2526): ec2:DescribeInstances, + # so it can verify each node's Auto Scaling group before applying the pool label. + node_pool_labeler = optional(bool, false) })) validation { @@ -236,13 +239,12 @@ variable "tags" { variable "enable_runtime_plugins" { description = <<-EOT - Create the exclusive gVisor runtime-plugin node group. Default false on EKS: - the pool identity keys on the node-restriction.kubernetes.io/shifter-pool - label, which NodeRestriction forbids a kubelet from self-registering. EKS - applies managed-node-group labels through the kubelet (unlike GKE, whose - control plane applies them via a trusted path), so the node fails to join - with that label. Enabling it requires a trusted node-labeler for EKS parity - with GKE (tracked follow-up). Keep false until that lands. + Create the exclusive gVisor runtime-plugin node group and its dedicated node + role. Pool identity keys on the node-restriction.kubernetes.io/shifter-pool + label, which NodeRestriction forbids a kubelet from setting; on EKS the + trusted node-pool labeler applies it to nodes in this pool's Auto Scaling + group (#2526). Enabling it requires a workload identity with node_pool_labeler for + that controller. EOT type = bool default = false diff --git a/scripts/bootstrap/aws_eks.py b/scripts/bootstrap/aws_eks.py index e1d5dcafb..3f33b510b 100644 --- a/scripts/bootstrap/aws_eks.py +++ b/scripts/bootstrap/aws_eks.py @@ -85,6 +85,7 @@ # Probed only when the environment defines the role (see _OPTIONAL_WORKLOAD_ROLE_KEYS). _OPTIONAL_IRSA_PROBE_IDENTITIES = { "artifactAcquirer": ("shifter-acquisition", "artifact-acquirer"), + "nodePoolLabeler": ("shifter-platform", "node-pool-labeler"), } _IRSA_PROBE_SCRIPT = """import json, os import boto3 @@ -120,8 +121,9 @@ _WORKLOAD_ROLE_KEYS = frozenset({"portal", "workers", "ctfScheduler", "provisionerLauncher", "provisioner", "migrator"}) # Workload roles an environment may define. Their presence enables the matching # capability: artifactAcquirer turns on isolated feature-artifact acquisition -# Jobs (ADR-034-R12, #2463). -_OPTIONAL_WORKLOAD_ROLE_KEYS = frozenset({"artifactAcquirer"}) +# Jobs (ADR-034-R12, #2463); nodePoolLabeler turns on the trusted runtime-plugin +# node-pool labeler (#2526). +_OPTIONAL_WORKLOAD_ROLE_KEYS = frozenset({"artifactAcquirer", "nodePoolLabeler"}) # Single source of truth in the installation package (installation.runtime_inventory_aws), # so the renderer and the backend bundle's generated-output projection cannot drift. _RENDERER_OWNED_RUNTIME_ENV = AWS_RENDERER_OWNED_RUNTIME_ENV_KEYS @@ -1007,6 +1009,25 @@ def _feature_artifact_job_image(roles: Mapping[str, str], validated_images: Mapp return validated_images["platform"] +def _runtime_plugin_pool_values(roles: Mapping[str, str], terraform_outputs: Mapping[str, object]) -> dict[str, object]: + """The trusted runtime-plugin node-pool labeler (#2526), enabled with the plugin pool. + + The labeler identity and the pool's Auto Scaling group come from the same + Terraform toggle, so exactly one of them present means a misconfigured + environment. + """ + raw = terraform_outputs.get("runtime_plugin_node_group_asg") + node_group_asg = raw.get("value", "") if isinstance(raw, Mapping) else "" + if not isinstance(node_group_asg, str): + raise ValueError("runtime_plugin_node_group_asg must be a string") + labeler = "nodePoolLabeler" in roles + if labeler != bool(node_group_asg): + raise ValueError("the runtime-plugin pool and the nodePoolLabeler identity must be enabled together") + if node_group_asg and not re.fullmatch(r"[\w+=,.@/:-]{1,255}", node_group_asg): + raise ValueError("runtime_plugin_node_group_asg must be an Auto Scaling group name") + return {"labeler": {"enabled": labeler, "nodeGroupAsg": node_group_asg}} + + def _edge_values(config: RootConfig, terraform_outputs: Mapping[str, object]) -> dict[str, object]: """Public edge: ALB ingress with ACM TLS and WAF, restricted to the edge client CIDRs.""" edge_client_cidrs = _cidr_output(terraform_outputs, "edge_client_cidrs") @@ -1131,6 +1152,7 @@ def render_aws_values( "provisioner": {"taskRunner": "aws"}, "edge": _edge_values(config, terraform_outputs), "featureArtifactAcquisition": _aws_feature_artifact_acquisition_values(terraform_outputs), + "runtimePluginPool": _runtime_plugin_pool_values(roles, terraform_outputs), "services": { service: {"annotations": {"alb.ingress.kubernetes.io/healthcheck-path": path}} for service, path in _ALB_HEALTH_CHECK_PATHS.items() diff --git a/scripts/bootstrap/tests/test_aws_eks.py b/scripts/bootstrap/tests/test_aws_eks.py index 30507bd6a..3ed797833 100644 --- a/scripts/bootstrap/tests/test_aws_eks.py +++ b/scripts/bootstrap/tests/test_aws_eks.py @@ -969,6 +969,31 @@ def runner(cmd, **kwargs): assert not any("supersecretvalue12345" in a for a in cmd) +def test_runtime_plugin_pool_labeler_follows_the_pool_toggle(): + """The trusted node-pool labeler (#2526) renders only with the plugin pool's node role.""" + disabled = aws_eks.render_aws_values(_config(), _terraform_outputs(), _images()) + assert disabled["runtimePluginPool"] == {"labeler": {"enabled": False, "nodeGroupAsg": ""}} + assert "nodePoolLabeler" not in disabled["identity"]["serviceAccountRoleArns"] + + outputs = _terraform_outputs() + labeler = "arn:aws:iam::123456789012:role/shifter-dev-node-pool-labeler" + node_group = "eks-runtime-plugins-1234-abcd" + outputs["workload_role_arns"]["value"]["nodePoolLabeler"] = labeler + outputs["runtime_plugin_node_group_asg"] = {"value": node_group} + enabled = aws_eks.render_aws_values(_config(), outputs, _images()) + assert enabled["runtimePluginPool"] == {"labeler": {"enabled": True, "nodeGroupAsg": node_group}} + assert enabled["identity"]["serviceAccountRoleArns"]["nodePoolLabeler"] == labeler + + # Exactly one of the identity and the pool role means a misconfigured environment. + del outputs["runtime_plugin_node_group_asg"] + with pytest.raises(ValueError, match="enabled together"): + aws_eks.render_aws_values(_config(), outputs, _images()) + pool_only = _terraform_outputs() + pool_only["runtime_plugin_node_group_asg"] = {"value": node_group} + with pytest.raises(ValueError, match="enabled together"): + aws_eks.render_aws_values(_config(), pool_only, _images()) + + def test_feature_artifact_acquisition_is_enabled_only_by_the_acquirer_identity(monkeypatch): """The artifactAcquirer role enables acquisition Jobs (#2463); without it they stay off.""" disabled = aws_eks.render_aws_values(_config(), _terraform_outputs(), _images()) diff --git a/shifter/shifter_platform/shared/cloud/eks_node_pool_labeler.py b/shifter/shifter_platform/shared/cloud/eks_node_pool_labeler.py new file mode 100644 index 000000000..19f276263 --- /dev/null +++ b/shifter/shifter_platform/shared/cloud/eks_node_pool_labeler.py @@ -0,0 +1,201 @@ +"""Trusted labeler for the EKS runtime-plugin node pool (#2526). + +Runtime-plugin worker Jobs select ``node-restriction.kubernetes.io/shifter-pool= +runtime-plugin``. NodeRestriction forbids a kubelet from setting labels under +that prefix, which is what makes the label a trustworthy pool identity: a node +cannot place itself into (or out of) the sandbox pool. GKE applies the label +through its control plane; EKS managed node groups apply labels through the +kubelet, so on EKS this controller applies it instead. + +A node qualifies only when the EC2 instance behind it is running, its private +DNS name equals the node name the EKS authenticator assigned, and its +``aws:autoscaling:groupName`` tag names the runtime-plugin node group's Auto +Scaling group. Only AWS can set ``aws:``-prefixed tags, and nothing on the node +can change its DNS name or state. A node that positively fails the check loses +the label; a provider error leaves the node unchanged. + +Usage: ``python -m shared.cloud.eks_node_pool_labeler`` with +``RUNTIME_PLUGIN_NODE_GROUP_ASG`` and ``AWS_REGION`` in the environment. +""" + +from __future__ import annotations + +import logging +import os +import re +import sys +import time +from pathlib import Path +from typing import Protocol, TypedDict + +logger = logging.getLogger(__name__) + +POOL_LABEL = "node-restriction.kubernetes.io/shifter-pool" +POOL_VALUE = "runtime-plugin" +ASG_TAG = "aws:autoscaling:groupName" +HEARTBEAT = Path("/tmp/node-pool-labeler-heartbeat") # noqa: S108 # nosec B108 # NOSONAR(S5443) +_PROVIDER_ID = re.compile(r"^aws:///[a-z0-9-]+/(i-[0-9a-f]{8,17})$") +_ASG_NAME = re.compile(r"[\w+=,.@/:-]{1,255}") +_POLL_SECONDS = 15 + + +class _Tag(TypedDict): + """One EC2 resource tag.""" + + Key: str + Value: str + + +class _InstanceState(TypedDict): + """The lifecycle state of a described instance.""" + + Name: str + + +class _Instance(TypedDict, total=False): + """The described-instance fields the labeler reads.""" + + State: _InstanceState + PrivateDnsName: str + Tags: list[_Tag] + + +class _Reservation(TypedDict): + """One DescribeInstances reservation.""" + + Instances: list[_Instance] + + +class _DescribeInstances(TypedDict, total=False): + """The DescribeInstances response fields the labeler reads.""" + + Reservations: list[_Reservation] + + +class Ec2Reader(Protocol): + """The one EC2 read the labeler needs.""" + + def describe_instances(self, **kwargs: list[str]) -> _DescribeInstances: ... + + +class NodeMetadata(Protocol): + """The Node metadata the labeler reads.""" + + name: str + labels: dict[str, str] | None + + +class NodeSpec(Protocol): + """The Node spec field the labeler reads.""" + + provider_id: str | None + + +class Node(Protocol): + """The fields of a Kubernetes Node the labeler reads.""" + + metadata: NodeMetadata + spec: NodeSpec + + +class NodeList(Protocol): + """A listing of Nodes.""" + + items: list[Node] + + +class NodeApi(Protocol): + """The Kubernetes Node operations the labeler uses.""" + + def list_node(self) -> NodeList: ... + + def patch_node(self, name: str, body: dict[str, object]) -> object: ... + + +def instance_group(ec2: Ec2Reader, instance_id: str, node_name: str) -> str: + """The Auto Scaling group of the running instance named ``node_name``, or ``""``. + + Empty when the instance does not exist, is not running, does not match the + node name, or belongs to no group. Provider errors other than a missing + instance propagate. + """ + from botocore.exceptions import ClientError + + try: + response = ec2.describe_instances(InstanceIds=[instance_id]) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") == "InvalidInstanceID.NotFound": + return "" + raise + instances = [instance for reservation in response.get("Reservations", []) for instance in reservation["Instances"]] + if len(instances) != 1: + return "" + instance = instances[0] + running = "State" in instance and instance["State"]["Name"] == "running" + named = instance.get("PrivateDnsName") == node_name + groups = [tag["Value"] for tag in instance.get("Tags", []) if tag["Key"] == ASG_TAG] + return groups[0] if running and named and len(groups) == 1 else "" + + +def node_qualifies(node: Node, ec2: Ec2Reader, pool_asg: str) -> bool | None: + """Whether ``node`` belongs to the pool; ``None`` when the provider could not answer.""" + from botocore.exceptions import BotoCoreError, ClientError + + match = _PROVIDER_ID.fullmatch(node.spec.provider_id or "") + if match is None: + return False + try: + return instance_group(ec2, match.group(1), node.metadata.name) == pool_asg + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "unknown") + logger.warning("node pool check deferred node=%s error=ClientError code=%s", node.metadata.name, code) + except BotoCoreError as exc: + logger.warning("node pool check deferred node=%s error=%s", node.metadata.name, type(exc).__name__) + return None + + +def reconcile_node(core: NodeApi, node: Node, ec2: Ec2Reader, pool_asg: str) -> None: + """Apply or remove the pool label so it matches the node's verified identity.""" + qualifies = node_qualifies(node, ec2, pool_asg) + if qualifies is None: + return + current = (node.metadata.labels or {}).get(POOL_LABEL) + if qualifies and current != POOL_VALUE: + core.patch_node(node.metadata.name, {"metadata": {"labels": {POOL_LABEL: POOL_VALUE}}}) + logger.info("node pool label applied node=%s", node.metadata.name) + elif not qualifies and current is not None: + core.patch_node(node.metadata.name, {"metadata": {"labels": {POOL_LABEL: None}}}) + logger.warning("node pool label removed from unverified node=%s", node.metadata.name) + + +def reconcile_all(core: NodeApi, ec2: Ec2Reader, pool_asg: str) -> None: + """One pass over every node.""" + for node in core.list_node().items: + reconcile_node(core, node, ec2, pool_asg) + + +def main() -> int: + """Run the labeler until the process is stopped; returns 2 on missing configuration.""" + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + pool_asg = os.environ.get("RUNTIME_PLUGIN_NODE_GROUP_ASG", "") + region = os.environ.get("AWS_REGION", "") + if not _ASG_NAME.fullmatch(pool_asg) or not region: + sys.stderr.write("RUNTIME_PLUGIN_NODE_GROUP_ASG and AWS_REGION are required\n") + return 2 + import boto3 + + from shared.cloud.kubernetes._client import load_kubernetes_api + + _batch, core, _client, api_exception = load_kubernetes_api() + ec2 = boto3.client("ec2", region_name=region) + while True: + try: + reconcile_all(core, ec2, pool_asg) + HEARTBEAT.touch() + except api_exception as exc: + logger.warning("node pool reconcile deferred status=%s", getattr(exc, "status", "unknown")) + time.sleep(_POLL_SECONDS) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/shifter/shifter_platform/tests/shared/cloud/test_eks_node_pool_labeler.py b/shifter/shifter_platform/tests/shared/cloud/test_eks_node_pool_labeler.py new file mode 100644 index 000000000..c580b49bb --- /dev/null +++ b/shifter/shifter_platform/tests/shared/cloud/test_eks_node_pool_labeler.py @@ -0,0 +1,114 @@ +"""Trusted EKS runtime-plugin pool labeler (#2526).""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +from botocore.exceptions import ClientError, EndpointConnectionError + +from shared.cloud.eks_node_pool_labeler import ASG_TAG, POOL_LABEL, POOL_VALUE, reconcile_all + +POOL_ASG = "eks-runtime-plugins-1234-abcd" +NODE = "ip-10-0-1-10.us-east-2.compute.internal" +PLATFORM_NODE = "ip-10-0-1-11.us-east-2.compute.internal" + + +class FakeEc2: + def __init__(self, instances: dict[str, dict]) -> None: + self.instances = instances + self.error: Exception | None = None + + def describe_instances(self, **kwargs): + if self.error: + raise self.error + instance = self.instances.get(kwargs["InstanceIds"][0]) + if instance is None: + raise ClientError({"Error": {"Code": "InvalidInstanceID.NotFound"}}, "DescribeInstances") + return {"Reservations": [{"Instances": [instance]}]} + + +class FakeCore: + def __init__(self, nodes: list) -> None: + self.nodes = nodes + self.patches: list[tuple[str, dict]] = [] + + def list_node(self): + return SimpleNamespace(items=self.nodes) + + def patch_node(self, name, body): + self.patches.append((name, body)) + + +def _node(name=NODE, instance="i-0abc12345678def00", labels=None): + return SimpleNamespace( + metadata=SimpleNamespace(name=name, labels=labels or {}), + spec=SimpleNamespace(provider_id=f"aws:///us-east-2a/{instance}" if instance else None), + ) + + +def _instance(group=POOL_ASG, dns=NODE, state="running"): + tags = [{"Key": "Name", "Value": "worker"}] + if group: + tags.append({"Key": ASG_TAG, "Value": group}) + return {"State": {"Name": state}, "PrivateDnsName": dns, "Tags": tags} + + +@pytest.fixture +def ec2(): + return FakeEc2( + { + "i-0abc12345678def00": _instance(), + "i-0abc12345678def01": _instance(group="eks-platform-9876", dns=PLATFORM_NODE), + "i-0abc12345678def02": _instance(dns="ip-10-0-9-9.us-east-2.compute.internal"), + "i-0abc12345678def03": _instance(state="stopped"), + "i-0abc12345678def04": _instance(group=""), + } + ) + + +def test_labels_only_nodes_in_the_pool_auto_scaling_group(ec2): + core = FakeCore([_node(), _node(name=PLATFORM_NODE, instance="i-0abc12345678def01")]) + + reconcile_all(core, ec2, POOL_ASG) + + assert core.patches == [(NODE, {"metadata": {"labels": {POOL_LABEL: POOL_VALUE}}})] + already = FakeCore([_node(labels={POOL_LABEL: POOL_VALUE})]) + reconcile_all(already, ec2, POOL_ASG) + assert already.patches == [] + + +def test_removes_the_label_from_any_node_that_fails_verification(ec2): + labeled = {POOL_LABEL: POOL_VALUE} + nodes = [ + _node(name=PLATFORM_NODE, instance="i-0abc12345678def01", labels=dict(labeled)), # other group + _node(name=NODE, instance="i-0abc12345678def02", labels=dict(labeled)), # DNS name mismatch + _node(name=NODE, instance="i-0abc12345678def03", labels=dict(labeled)), # stopped + _node(name=NODE, instance="i-0abc12345678def04", labels=dict(labeled)), # no group tag + _node(name="ip-b", instance="i-0abc12345678def99", labels=dict(labeled)), # missing instance + _node(name="ip-c", instance=None, labels=dict(labeled)), # no provider ID + ] + core = FakeCore(nodes) + + reconcile_all(core, ec2, POOL_ASG) + + assert len(core.patches) == len(nodes) + assert all(body == {"metadata": {"labels": {POOL_LABEL: None}}} for _, body in core.patches) + + +@pytest.mark.parametrize( + "error", + [ + EndpointConnectionError(endpoint_url="https://ec2.us-east-2.amazonaws.com"), + ClientError({"Error": {"Code": "UnauthorizedOperation"}}, "DescribeInstances"), + ], +) +def test_provider_errors_leave_nodes_unchanged(ec2, error, caplog): + ec2.error = error + core = FakeCore([_node(), _node(name="ip-x", labels={POOL_LABEL: POOL_VALUE})]) + + reconcile_all(core, ec2, POOL_ASG) + + assert core.patches == [] + if isinstance(error, ClientError): + assert "code=UnauthorizedOperation" in caplog.text