From b14583cab7a540234520c72a8d3b1cc8801b5a3a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Nov 2026 17:25:35 -0700 Subject: [PATCH 1/2] Add azure_aks_workload_security: pod security contexts and network policies from the AKS Kubernetes API Reads each Entra-integrated AKS cluster's Kubernetes API with a token for the AKS server application, using the API server address and CA certificate from listClusterUserCredential; no kubectl and no stored kubeconfig. Pods are grouped into workloads, and each container's effective security context is recorded: non-root, privilege escalation, read-only root filesystem, capabilities, privileged and seccomp profile, with host namespaces and host path volumes. Namespaces carry their Pod Security Admission levels and network policy coverage, and the cluster whether a policy engine enforces those policies. System namespaces and Windows workloads are listed but not judged. A list the API refuses leaves its counts null and fails the run; an API server that does not answer is tried once and reported as unreachable; a stopped cluster reads as running nothing. AzureRestClient gains get_url for a GET outside ARM. A workload running two pod templates, as during a rollout, is represented by its least restricted one. A runAsUser of 0 is root whatever runAsNonRoot says. Windows pods are recognised by the os field or the node selector. No token is sent to an API server address that is not https. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01D7SnHUNrjyxciAGKLGQrgq --- CHANGELOG.md | 14 + docs/ksi_mapping.md | 16 +- fetchers/azure/README.md | 4 +- fetchers/azure/_shared/azure_rest.py | 8 + .../azure/aks_workload_security/fetcher.py | 554 +++++++++++++++++ .../azure/aks_workload_security/fetcher.yaml | 143 +++++ tests/test_azure_aks_workload_security.py | 558 ++++++++++++++++++ tests/test_contracts.py | 1 + 8 files changed, 1290 insertions(+), 8 deletions(-) create mode 100644 fetchers/azure/aks_workload_security/fetcher.py create mode 100644 fetchers/azure/aks_workload_security/fetcher.yaml create mode 100644 tests/test_azure_aks_workload_security.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ee79711..8b7e8b24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -412,6 +412,20 @@ schemas and the `paramify` CLI — not the internal code. - Needs read on the workspace's logs (Reader, or a custom role with `workspaces/query/read` on the four tables, plus `IdentityInfo` when `groups` is set). Maps to KSI-IAM-APM, KSI-SVC-SIN, KSI-IAM-SNU and KSI-MLA-LET. +- **`azure_aks_workload_security`** (`EVD-AZURE-AKS-WORKLOAD-SECURITY`): workload security + read from each AKS cluster's Kubernetes API with a Microsoft Entra token, with no + kubectl and no stored kubeconfig. For every workload (pods grouped by the Deployment, + StatefulSet, DaemonSet or Job that runs them) it records each container's effective + security context: non-root, privilege escalation, read-only root filesystem, dropped and + added capabilities, privileged, seccomp profile, plus host namespaces and host path + volumes. For every namespace it records the Pod Security Admission levels and network + policy coverage, including default-deny policies and workloads no policy selects, and + for the cluster whether a network policy engine enforces those policies at all. + System namespaces (`AKS_SYSTEM_NAMESPACES`) and Windows workloads are listed but not + judged. Needs Azure Kubernetes Service Cluster User Role and Azure Kubernetes Service + RBAC Reader on the cluster, Entra integration, and a network path to the API server; a + list that cannot be read leaves its counts null and fails the run. No environment + variables, images, secrets or logs are read. - **`checkov_bicep` 0.3.0: the TFVC source signs in with Entra when no token is set.** Before, `source_type: tfvc` authenticated only with the `tfvc_token` PAT. - With no `tfvc_token`, the fetcher gets an Entra token for Azure DevOps from diff --git a/docs/ksi_mapping.md b/docs/ksi_mapping.md index e1bcc72e..1cfb2e8f 100644 --- a/docs/ksi_mapping.md +++ b/docs/ksi_mapping.md @@ -40,7 +40,7 @@ python tools/gen_ksi_mapping.py *Controls:* `cm-2`, `si-3` -*9 fetchers:* [`aws_organizations_scp`](../fetchers/aws/organizations_scp), [`azure_app_service_configuration`](../fetchers/azure/app_service_configuration), [`azure_network_security_groups`](../fetchers/azure/network_security_groups), [`azure_rbac_custom_roles`](../fetchers/azure/rbac_custom_roles), [`azure_rbac_role_assignments`](../fetchers/azure/rbac_role_assignments), [`azure_resource_locks`](../fetchers/azure/resource_locks), [`azure_sql_managed_instance_configuration`](../fetchers/azure/sql_managed_instance_configuration), [`azure_vnet_egress_routing`](../fetchers/azure/vnet_egress_routing), [`k8s_kubectl_security`](../fetchers/k8s/kubectl_security) +*10 fetchers:* [`aws_organizations_scp`](../fetchers/aws/organizations_scp), [`azure_aks_workload_security`](../fetchers/azure/aks_workload_security), [`azure_app_service_configuration`](../fetchers/azure/app_service_configuration), [`azure_network_security_groups`](../fetchers/azure/network_security_groups), [`azure_rbac_custom_roles`](../fetchers/azure/rbac_custom_roles), [`azure_rbac_role_assignments`](../fetchers/azure/rbac_role_assignments), [`azure_resource_locks`](../fetchers/azure/resource_locks), [`azure_sql_managed_instance_configuration`](../fetchers/azure/sql_managed_instance_configuration), [`azure_vnet_egress_routing`](../fetchers/azure/vnet_egress_routing), [`k8s_kubectl_security`](../fetchers/k8s/kubectl_security) #### `KSI-CNA-EIS` — Enforcing Intended State *(optional at Low)* @@ -64,7 +64,7 @@ python tools/gen_ksi_mapping.py *Controls:* `ac-17.3`, `ac-18.1`, `ac-18.3`, `ac-20.1`, `ca-9`, `sc-7.3`, `sc-7.4`, `sc-7.5`, `sc-7.8`, `sc-8`, `sc-10`, `si-10`, `si-11`, `si-16` -*39 fetchers:* [`aws_ec2_public_exposure`](../fetchers/aws/ec2_public_exposure), [`aws_security_groups`](../fetchers/aws/security_groups), [`azure_ai_hub_configuration`](../fetchers/azure/ai_hub_configuration), [`azure_ai_services_configuration`](../fetchers/azure/ai_services_configuration), [`azure_aks_cluster_configuration`](../fetchers/azure/aks_cluster_configuration), [`azure_api_management_configuration`](../fetchers/azure/api_management_configuration), [`azure_app_gateway_tls_listeners`](../fetchers/azure/app_gateway_tls_listeners), [`azure_app_gateway_waf`](../fetchers/azure/app_gateway_waf), [`azure_app_service_configuration`](../fetchers/azure/app_service_configuration), [`azure_bastion_configuration`](../fetchers/azure/bastion_configuration), [`azure_container_registry_configuration`](../fetchers/azure/container_registry_configuration), [`azure_container_workloads_configuration`](../fetchers/azure/container_workloads_configuration), [`azure_data_services_configuration`](../fetchers/azure/data_services_configuration), [`azure_databricks_workspace_configuration`](../fetchers/azure/databricks_workspace_configuration), [`azure_dns_configuration`](../fetchers/azure/dns_configuration), [`azure_firewall_configuration`](../fetchers/azure/firewall_configuration), [`azure_front_door_tls`](../fetchers/azure/front_door_tls), [`azure_front_door_waf_coverage`](../fetchers/azure/front_door_waf_coverage), [`azure_front_door_waf_policies`](../fetchers/azure/front_door_waf_policies), [`azure_function_app_configuration`](../fetchers/azure/function_app_configuration), [`azure_key_vault_configuration`](../fetchers/azure/key_vault_configuration), [`azure_mysql_configuration`](../fetchers/azure/mysql_configuration), [`azure_network_security_groups`](../fetchers/azure/network_security_groups), [`azure_private_endpoint_configuration`](../fetchers/azure/private_endpoint_configuration), [`azure_sql_managed_instance_configuration`](../fetchers/azure/sql_managed_instance_configuration), [`azure_sql_server_configuration`](../fetchers/azure/sql_server_configuration), [`azure_vm_hardening_status`](../fetchers/azure/vm_hardening_status), [`crowdstrike_firewall_policies`](../fetchers/crowdstrike/firewall_policies), [`gcp_cloud_sql_network_configuration`](../fetchers/gcp/cloud_sql_network_configuration), [`gcp_compute_instance_configuration`](../fetchers/gcp/compute_instance_configuration), [`gcp_firewall_rules`](../fetchers/gcp/firewall_rules), [`gcp_gke_cluster_configuration`](../fetchers/gcp/gke_cluster_configuration), [`github_actions_security`](../fetchers/github/actions_security), [`k8s_eks_microservice_segmentation`](../fetchers/k8s/eks_microservice_segmentation), [`k8s_kubectl_security`](../fetchers/k8s/kubectl_security), [`oci_compute_instances`](../fetchers/oci/compute_instances), [`oci_data_service_exposure`](../fetchers/oci/data_service_exposure), [`oci_network_exposure`](../fetchers/oci/network_exposure), [`wiz_attack_surface_findings`](../fetchers/wiz/attack_surface_findings) +*40 fetchers:* [`aws_ec2_public_exposure`](../fetchers/aws/ec2_public_exposure), [`aws_security_groups`](../fetchers/aws/security_groups), [`azure_ai_hub_configuration`](../fetchers/azure/ai_hub_configuration), [`azure_ai_services_configuration`](../fetchers/azure/ai_services_configuration), [`azure_aks_cluster_configuration`](../fetchers/azure/aks_cluster_configuration), [`azure_aks_workload_security`](../fetchers/azure/aks_workload_security), [`azure_api_management_configuration`](../fetchers/azure/api_management_configuration), [`azure_app_gateway_tls_listeners`](../fetchers/azure/app_gateway_tls_listeners), [`azure_app_gateway_waf`](../fetchers/azure/app_gateway_waf), [`azure_app_service_configuration`](../fetchers/azure/app_service_configuration), [`azure_bastion_configuration`](../fetchers/azure/bastion_configuration), [`azure_container_registry_configuration`](../fetchers/azure/container_registry_configuration), [`azure_container_workloads_configuration`](../fetchers/azure/container_workloads_configuration), [`azure_data_services_configuration`](../fetchers/azure/data_services_configuration), [`azure_databricks_workspace_configuration`](../fetchers/azure/databricks_workspace_configuration), [`azure_dns_configuration`](../fetchers/azure/dns_configuration), [`azure_firewall_configuration`](../fetchers/azure/firewall_configuration), [`azure_front_door_tls`](../fetchers/azure/front_door_tls), [`azure_front_door_waf_coverage`](../fetchers/azure/front_door_waf_coverage), [`azure_front_door_waf_policies`](../fetchers/azure/front_door_waf_policies), [`azure_function_app_configuration`](../fetchers/azure/function_app_configuration), [`azure_key_vault_configuration`](../fetchers/azure/key_vault_configuration), [`azure_mysql_configuration`](../fetchers/azure/mysql_configuration), [`azure_network_security_groups`](../fetchers/azure/network_security_groups), [`azure_private_endpoint_configuration`](../fetchers/azure/private_endpoint_configuration), [`azure_sql_managed_instance_configuration`](../fetchers/azure/sql_managed_instance_configuration), [`azure_sql_server_configuration`](../fetchers/azure/sql_server_configuration), [`azure_vm_hardening_status`](../fetchers/azure/vm_hardening_status), [`crowdstrike_firewall_policies`](../fetchers/crowdstrike/firewall_policies), [`gcp_cloud_sql_network_configuration`](../fetchers/gcp/cloud_sql_network_configuration), [`gcp_compute_instance_configuration`](../fetchers/gcp/compute_instance_configuration), [`gcp_firewall_rules`](../fetchers/gcp/firewall_rules), [`gcp_gke_cluster_configuration`](../fetchers/gcp/gke_cluster_configuration), [`github_actions_security`](../fetchers/github/actions_security), [`k8s_eks_microservice_segmentation`](../fetchers/k8s/eks_microservice_segmentation), [`k8s_kubectl_security`](../fetchers/k8s/kubectl_security), [`oci_compute_instances`](../fetchers/oci/compute_instances), [`oci_data_service_exposure`](../fetchers/oci/data_service_exposure), [`oci_network_exposure`](../fetchers/oci/network_exposure), [`wiz_attack_surface_findings`](../fetchers/wiz/attack_surface_findings) #### `KSI-CNA-OFA` — Optimizing for Availability @@ -94,7 +94,7 @@ python tools/gen_ksi_mapping.py *Controls:* `ac-12`, `ac-17.3`, `ca-9`, `sc-4`, `sc-7`, `sc-7.7`, `sc-8`, `sc-10` -*20 fetchers:* [`aws_network_acls`](../fetchers/aws/network_acls), [`aws_network_firewall_rules`](../fetchers/aws/network_firewall_rules), [`aws_vpc_network_segmentation`](../fetchers/aws/vpc_network_segmentation), [`azure_aks_cluster_configuration`](../fetchers/azure/aks_cluster_configuration), [`azure_app_service_configuration`](../fetchers/azure/app_service_configuration), [`azure_container_workloads_configuration`](../fetchers/azure/container_workloads_configuration), [`azure_databricks_workspace_configuration`](../fetchers/azure/databricks_workspace_configuration), [`azure_firewall_configuration`](../fetchers/azure/firewall_configuration), [`azure_network_security_groups`](../fetchers/azure/network_security_groups), [`azure_network_traffic_activity`](../fetchers/azure/network_traffic_activity), [`azure_private_dns_zone_configuration`](../fetchers/azure/private_dns_zone_configuration), [`azure_private_endpoint_configuration`](../fetchers/azure/private_endpoint_configuration), [`azure_sentinel_incidents`](../fetchers/azure/sentinel_incidents), [`azure_virtual_wan_configuration`](../fetchers/azure/virtual_wan_configuration), [`azure_vnet_egress_routing`](../fetchers/azure/vnet_egress_routing), [`gcp_gke_cluster_configuration`](../fetchers/gcp/gke_cluster_configuration), [`gcp_vpc_network_configuration`](../fetchers/gcp/vpc_network_configuration), [`k8s_eks_microservice_segmentation`](../fetchers/k8s/eks_microservice_segmentation), [`k8s_kubectl_security`](../fetchers/k8s/kubectl_security), [`oci_zpr_policies`](../fetchers/oci/zpr_policies) +*21 fetchers:* [`aws_network_acls`](../fetchers/aws/network_acls), [`aws_network_firewall_rules`](../fetchers/aws/network_firewall_rules), [`aws_vpc_network_segmentation`](../fetchers/aws/vpc_network_segmentation), [`azure_aks_cluster_configuration`](../fetchers/azure/aks_cluster_configuration), [`azure_aks_workload_security`](../fetchers/azure/aks_workload_security), [`azure_app_service_configuration`](../fetchers/azure/app_service_configuration), [`azure_container_workloads_configuration`](../fetchers/azure/container_workloads_configuration), [`azure_databricks_workspace_configuration`](../fetchers/azure/databricks_workspace_configuration), [`azure_firewall_configuration`](../fetchers/azure/firewall_configuration), [`azure_network_security_groups`](../fetchers/azure/network_security_groups), [`azure_network_traffic_activity`](../fetchers/azure/network_traffic_activity), [`azure_private_dns_zone_configuration`](../fetchers/azure/private_dns_zone_configuration), [`azure_private_endpoint_configuration`](../fetchers/azure/private_endpoint_configuration), [`azure_sentinel_incidents`](../fetchers/azure/sentinel_incidents), [`azure_virtual_wan_configuration`](../fetchers/azure/virtual_wan_configuration), [`azure_vnet_egress_routing`](../fetchers/azure/vnet_egress_routing), [`gcp_gke_cluster_configuration`](../fetchers/gcp/gke_cluster_configuration), [`gcp_vpc_network_configuration`](../fetchers/gcp/vpc_network_configuration), [`k8s_eks_microservice_segmentation`](../fetchers/k8s/eks_microservice_segmentation), [`k8s_kubectl_security`](../fetchers/k8s/kubectl_security), [`oci_zpr_policies`](../fetchers/oci/zpr_policies) ### SVC — Service Configuration (7/7) @@ -330,7 +330,7 @@ python tools/gen_ksi_mapping.py *Controls:* `cm-2.2`, `cm-7.5`, `cm-8`, `cm-8.1`, `cm-12`, `cm-12.1`, `cp-2.8` -*19 fetchers:* [`aws_detect_new_aws_resource`](../fetchers/aws/detect_new_aws_resource), [`aws_resource_inventory`](../fetchers/aws/resource_inventory), [`azure_entra_service_principals`](../fetchers/azure/entra_service_principals), [`azure_resource_inventory`](../fetchers/azure/resource_inventory), [`azure_sql_managed_instance_configuration`](../fetchers/azure/sql_managed_instance_configuration), [`azure_storage_encryption_status`](../fetchers/azure/storage_encryption_status), [`crowdstrike_hosts`](../fetchers/crowdstrike/hosts), [`datadog_agent_hosts`](../fetchers/datadog/agent_hosts), [`datadog_apm_services`](../fetchers/datadog/apm_services), [`datadog_containers`](../fetchers/datadog/containers), [`github_repository_inventory`](../fetchers/github/repository_inventory), [`gitlab_project_summary`](../fetchers/gitlab/project_summary), [`k8s_eks_pod_inventory`](../fetchers/k8s/eks_pod_inventory), [`rippling_devices`](../fetchers/rippling/devices), [`sentinelone_agents`](../fetchers/sentinelone/agents), [`sentinelone_xdr_assets`](../fetchers/sentinelone/xdr_assets), [`sophos_endpoint_inventory`](../fetchers/sophos/endpoint_inventory), [`wiz_inventory`](../fetchers/wiz/inventory), [`wiz_scan_coverage`](../fetchers/wiz/scan_coverage) +*20 fetchers:* [`aws_config_inventory`](../fetchers/aws/config_inventory), [`aws_detect_new_aws_resource`](../fetchers/aws/detect_new_aws_resource), [`aws_resource_inventory`](../fetchers/aws/resource_inventory), [`azure_entra_service_principals`](../fetchers/azure/entra_service_principals), [`azure_resource_inventory`](../fetchers/azure/resource_inventory), [`azure_sql_managed_instance_configuration`](../fetchers/azure/sql_managed_instance_configuration), [`azure_storage_encryption_status`](../fetchers/azure/storage_encryption_status), [`crowdstrike_hosts`](../fetchers/crowdstrike/hosts), [`datadog_agent_hosts`](../fetchers/datadog/agent_hosts), [`datadog_apm_services`](../fetchers/datadog/apm_services), [`datadog_containers`](../fetchers/datadog/containers), [`github_repository_inventory`](../fetchers/github/repository_inventory), [`gitlab_project_summary`](../fetchers/gitlab/project_summary), [`k8s_eks_pod_inventory`](../fetchers/k8s/eks_pod_inventory), [`rippling_devices`](../fetchers/rippling/devices), [`sentinelone_agents`](../fetchers/sentinelone/agents), [`sentinelone_xdr_assets`](../fetchers/sentinelone/xdr_assets), [`sophos_endpoint_inventory`](../fetchers/sophos/endpoint_inventory), [`wiz_inventory`](../fetchers/wiz/inventory), [`wiz_scan_coverage`](../fetchers/wiz/scan_coverage) #### `KSI-PIY-RES` — Reviewing Executive Support @@ -418,9 +418,9 @@ python tools/gen_ksi_mapping.py ## By fetcher -330 of 346 fetchers carry a mapping. +332 of 348 fetchers carry a mapping. -### aws (82) +### aws (83) | Fetcher | Indicators | |---|---| @@ -441,6 +441,7 @@ python tools/gen_ksi_mapping.py | [`aws_codepipeline_config`](../fetchers/aws/codepipeline_config) | `KSI-CMT-RMV`, `KSI-CMT-VTD` | | [`aws_component_ssl_enforcement_status`](../fetchers/aws/component_ssl_enforcement_status) | `KSI-SVC-SIN` | | [`aws_config_conformance_packs`](../fetchers/aws/config_conformance_packs) | `KSI-CNA-IBP`, `KSI-MLA-EVC` | +| [`aws_config_inventory`](../fetchers/aws/config_inventory) | `KSI-PIY-GIV` | | [`aws_config_monitoring`](../fetchers/aws/config_monitoring) | `KSI-CMT-LMC`, `KSI-MLA-EVC`, `KSI-SVC-ACM` | | [`aws_database_high_availability`](../fetchers/aws/database_high_availability) | `KSI-CNA-OFA` | | [`aws_detect_new_aws_resource`](../fetchers/aws/detect_new_aws_resource) | `KSI-CMT-LMC`, `KSI-PIY-GIV` | @@ -507,7 +508,7 @@ python tools/gen_ksi_mapping.py | [`aws_waf_all_rules`](../fetchers/aws/waf_all_rules) | `KSI-CNA-RVP` | | [`aws_waf_dos_rules`](../fetchers/aws/waf_dos_rules) | `KSI-CNA-RVP` | -### azure (104) +### azure (105) | Fetcher | Indicators | |---|---| @@ -515,6 +516,7 @@ python tools/gen_ksi_mapping.py | [`azure_ai_hub_configuration`](../fetchers/azure/ai_hub_configuration) | `KSI-CNA-MAT`, `KSI-CNA-RNT`, `KSI-IAM-SNU`, `KSI-MLA-LET`, `KSI-SVC-SIN` | | [`azure_ai_services_configuration`](../fetchers/azure/ai_services_configuration) | `KSI-CNA-MAT`, `KSI-IAM-SNU`, `KSI-MLA-LET`, `KSI-SVC-ASM`, `KSI-SVC-SIN` | | [`azure_aks_cluster_configuration`](../fetchers/azure/aks_cluster_configuration) | `KSI-CNA-MAT`, `KSI-CNA-RNT`, `KSI-CNA-ULN`, `KSI-IAM-ELP`, `KSI-IAM-SNU`, `KSI-SVC-EIS` | +| [`azure_aks_workload_security`](../fetchers/azure/aks_workload_security) | `KSI-CNA-DFP`, `KSI-CNA-MAT`, `KSI-CNA-ULN` | | [`azure_alert_rules`](../fetchers/azure/alert_rules) | `KSI-MLA-RVL` | | [`azure_api_management_configuration`](../fetchers/azure/api_management_configuration) | `KSI-CNA-MAT`, `KSI-CNA-RNT`, `KSI-SVC-ASM`, `KSI-SVC-SIN` | | [`azure_app_gateway_backend_tls`](../fetchers/azure/app_gateway_backend_tls) | `KSI-SVC-SIN`, `KSI-SVC-VCM` | diff --git a/fetchers/azure/README.md b/fetchers/azure/README.md index 3fd22e52..d0c31349 100644 --- a/fetchers/azure/README.md +++ b/fetchers/azure/README.md @@ -1,7 +1,7 @@ # Azure Azure fetchers pull evidence from your Azure subscriptions, Microsoft Entra ID and -Azure DevOps with Microsoft's Python SDKs and REST APIs. All 102 fetchers are +Azure DevOps with Microsoft's Python SDKs and REST APIs. All 104 fetchers are read-only and sign in with azure-identity's `DefaultAzureCredential`, so no key file or stored secret is required. Install the SDKs with `pip install -e '.[azure]'` (see [Install](../../README.md#install)). @@ -68,6 +68,7 @@ otherwise. | Access | Needed by | If it is missing | |---|---|---| +| Azure Kubernetes Service Cluster User Role and Azure Kubernetes Service RBAC Reader on each cluster, and a network path to its API server | `azure_aks_workload_security` | The cluster's workload fields read null and the run fails | | A custom role with `Microsoft.Web/sites/host/listkeys/action` and `Microsoft.Web/sites/config/list/action` | `azure_function_app_configuration` | Access keys and app settings read `not_authorized` per app; the run succeeds | | `Microsoft.Web/sites/config/list/action` (optional) | `azure_app_service_backups` | The custom backup configuration reads `not_authorized`; the run succeeds | | Key Vault Reader (RBAC vaults), or an access policy with key List and Get Rotation Policy (optional) | `azure_key_vault_key_rotation` | The vault reports `data_plane_status: inaccessible`; rotation policies still come from the management plane | @@ -282,6 +283,7 @@ Compute, containers and apps: |---|---| | `azure_vm_hardening_status` | VMs and scale sets: Trusted Launch, encryption at host, managed disks, SSH-key-only Linux sign-in, source image | | `azure_aks_cluster_configuration` | AKS control planes: private cluster or authorized IP ranges, Kubernetes RBAC and Entra integration, local accounts, workload identity, network policy | +| `azure_aks_workload_security` | AKS workloads read from each cluster's Kubernetes API: container security contexts (non-root, privilege escalation, read-only root filesystem, capabilities, privileged), host namespaces and host paths, Pod Security Admission levels per namespace, and network policies with default-deny and uncovered workloads | | `azure_container_registry_configuration` | Container registries: admin account, anonymous pull, network access and private endpoints | | `azure_container_workloads_configuration` | Container Apps environments, apps and jobs, and Container Instances: exposure, encryption between containers, identities, logging | | `azure_app_service_configuration` | Web apps: HTTPS-only, minimum TLS, FTP state, client certificates, App Service Authentication, managed identity, network access | diff --git a/fetchers/azure/_shared/azure_rest.py b/fetchers/azure/_shared/azure_rest.py index 34e17c02..db289895 100644 --- a/fetchers/azure/_shared/azure_rest.py +++ b/fetchers/azure/_shared/azure_rest.py @@ -197,6 +197,14 @@ def _follow(self, body: Dict[str, Any]) -> List[Dict[str, Any]]: items.extend(body.get("value") or []) return items + def get_url(self, url: str, resource: str, params: Optional[Dict[str, str]] = None, verify: Any = None): + """A GET to a service outside ARM that takes an Entra token for `resource`, such as an AKS cluster's Kubernetes + API; `verify` is the CA bundle path for a server whose certificate a private CA signed. The raw response.""" + kwargs: Dict[str, Any] = {"params": params, "timeout": HTTP_TIMEOUT} + if verify is not None: + kwargs["verify"] = verify + return self._send("get", url, resource, **kwargs) + def post(self, path: str, api_version: str) -> Dict[str, Any]: """An ARM read action that takes no body, such as fetchEventDetails.""" response = self._send("post", f"{self.arm}{path}", self.arm, params={"api-version": api_version}, timeout=HTTP_TIMEOUT) diff --git a/fetchers/azure/aks_workload_security/fetcher.py b/fetchers/azure/aks_workload_security/fetcher.py new file mode 100644 index 00000000..31e4cc13 --- /dev/null +++ b/fetchers/azure/aks_workload_security/fetcher.py @@ -0,0 +1,554 @@ +#!/usr/bin/env python3 +"""AKS workload security: pod and container security contexts, Pod Security Admission labels and network policies, read from each cluster's Kubernetes API with an Entra token.""" + +import base64 +import json +import logging +import os +import sys +import tempfile +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Tuple + +SCRIPT_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(SCRIPT_DIR.parent / "_shared")) +from azure_common import ( # noqa: E402 + Collector, + arm_id_key, + coverage_percentage, + dig, + resource_group_from_id, +) +from azure_rest import ArmError, int_config, run_subscription # noqa: E402 +from resource_exclusions import ResourceExclusions # noqa: E402 + +NAME = "azure_aks_workload_security" +logger = logging.getLogger(NAME) + +AKS_API = "2024-09-01" +# The AKS Microsoft Entra server application: the token audience of every Entra-integrated cluster, in every cloud. +AKS_SERVER_APP = "6dae42f8-4368-4678-94ff-3960e28e3630" +PAGE_SIZE = "500" +SYSTEM_NAMESPACES_ENV = "AKS_SYSTEM_NAMESPACES" +MAX_ROWS_ENV = "AKS_WORKLOAD_MAX_ROWS" +DEFAULT_MAX_ROWS = 500 +# Namespaces AKS and its add-ons run in; their pods need host access and are counted apart from the workloads. +DEFAULT_SYSTEM_NAMESPACES = ( + "kube-system", "kube-public", "kube-node-lease", "gatekeeper-system", "calico-system", "tigera-operator", + "app-routing-system", "aks-command", +) +PSA_PREFIX = "pod-security.kubernetes.io/" +PSA_ENFORCING = ("baseline", "restricted") +SECCOMP_CONFINED = ("runtimedefault", "localhost") +FINISHED_PHASES = ("succeeded", "failed") +NO_POLICY_ENGINE = ("", "none") +STOPPED = "stopped" + + +class KubeError(Exception): + """A Kubernetes API refusal; the status sits in the message the way classify_failure_code reads it.""" + + def __init__(self, status: int, message: str): + super().__init__(f"({status}) {message}") + self.status = status + + +def _lower(value: Any) -> str: + return str(value or "").lower() + + +def settings_from_env(collector: Optional[Collector] = None) -> Dict[str, Any]: + raw = os.environ.get(SYSTEM_NAMESPACES_ENV) + names = sorted({n.strip().lower() for n in (raw or "").replace(",", " ").split() if n.strip()}) + if raw is not None and raw.strip() and not names and collector is not None: + collector.record(f"config {SYSTEM_NAMESPACES_ENV}", ValueError( + f"invalid setting {SYSTEM_NAMESPACES_ENV}: it names no namespace")) + return { + "system_namespaces": names if (raw is not None and raw.strip()) else sorted(DEFAULT_SYSTEM_NAMESPACES), + "max_rows": int_config(MAX_ROWS_ENV, DEFAULT_MAX_ROWS), + } + + +# --- label selectors --- + +def selector_matches(selector: Optional[Dict[str, Any]], labels: Dict[str, str]) -> bool: + """A Kubernetes label selector against a pod's labels; an empty selector matches every pod.""" + selector = selector or {} + for key, value in (selector.get("matchLabels") or {}).items(): + if labels.get(key) != value: + return False + for expr in selector.get("matchExpressions") or []: + key, op, values = expr.get("key"), _lower(expr.get("operator")), expr.get("values") or [] + if op == "in" and labels.get(key) not in values: + return False + if op == "notin" and key in labels and labels[key] in values: + return False + if op == "exists" and key not in labels: + return False + if op == "doesnotexist" and key in labels: + return False + if op not in ("in", "notin", "exists", "doesnotexist"): + return False # an operator this fetcher does not know selects nothing, so no pod reads as covered by it + return True + + +def selector_is_empty(selector: Optional[Dict[str, Any]]) -> bool: + selector = selector or {} + return not (selector.get("matchLabels") or {}) and not (selector.get("matchExpressions") or []) + + +# --- containers and workloads --- + +def container_record(raw: Dict[str, Any], pod_context: Dict[str, Any], init: bool) -> Dict[str, Any]: + """One container's effective security context: its own setting, else the pod's where the pod can set it.""" + ctx = raw.get("securityContext") or {} + + def effective(key: str) -> Any: + return ctx[key] if ctx.get(key) is not None else pod_context.get(key) + + capabilities = ctx.get("capabilities") or {} + dropped = [str(c).upper() for c in capabilities.get("drop") or []] + added = sorted(str(c).upper() for c in capabilities.get("add") or []) + run_as_user = effective("runAsUser") + uid = run_as_user if isinstance(run_as_user, int) and not isinstance(run_as_user, bool) else None + seccomp = _lower(dig(ctx, "seccompProfile", "type") or dig(pod_context, "seccompProfile", "type")) or None + record = { + "name": raw.get("name"), + "init_container": init, + "run_as_non_root": effective("runAsNonRoot") is True, + "run_as_user": run_as_user, + # A runAsUser decides the user outright, so 0 is root whatever runAsNonRoot says; with no runAsUser, + # runAsNonRoot makes the kubelet refuse an image that would run as root. + "non_root_enforced": uid > 0 if uid is not None else effective("runAsNonRoot") is True, + "privilege_escalation_disabled": ctx.get("allowPrivilegeEscalation") is False, + "read_only_root_filesystem": ctx.get("readOnlyRootFilesystem") is True, + "drops_all_capabilities": "ALL" in dropped, + "added_capabilities": added, + "privileged": ctx.get("privileged") is True, + "seccomp_profile": seccomp, + } + record["hardened"] = (record["non_root_enforced"] and record["privilege_escalation_disabled"] + and record["read_only_root_filesystem"] and record["drops_all_capabilities"] + and not record["privileged"] and not added) + return record + + +def workload_of(pod: Dict[str, Any]) -> Tuple[str, str]: + """(kind, name) of what runs the pod: its controller, with a ReplicaSet read back to its Deployment.""" + meta = pod.get("metadata") or {} + owner = next((o for o in meta.get("ownerReferences") or [] if o.get("controller")), None) \ + or next(iter(meta.get("ownerReferences") or []), None) + if owner is None: + return "Pod", str(meta.get("name")) + kind, name = str(owner.get("kind")), str(owner.get("name")) + template_hash = (meta.get("labels") or {}).get("pod-template-hash") + if kind == "ReplicaSet" and template_hash and name.endswith(f"-{template_hash}"): + return "Deployment", name[: -len(template_hash) - 1] + return kind, name + + +def pod_record(pod: Dict[str, Any]) -> Dict[str, Any]: + meta, spec = pod.get("metadata") or {}, pod.get("spec") or {} + pod_context = spec.get("securityContext") or {} + containers = [container_record(c, pod_context, False) for c in spec.get("containers") or []] \ + + [container_record(c, pod_context, True) for c in spec.get("initContainers") or []] + kind, name = workload_of(pod) + host_paths = sorted({str(dig(v, "hostPath", "path")) for v in spec.get("volumes") or [] if v.get("hostPath")}) + return { + "namespace": meta.get("namespace"), + "name": meta.get("name"), + "labels": {str(k): str(v) for k, v in (meta.get("labels") or {}).items()}, + "workload_kind": kind, + "workload_name": name, + # Older manifests name the OS only through the node selector. + "windows": "windows" in (_lower(dig(spec, "os", "name")), _lower((spec.get("nodeSelector") or {}).get("kubernetes.io/os")), + _lower((spec.get("nodeSelector") or {}).get("beta.kubernetes.io/os"))), + "host_network": spec.get("hostNetwork") is True, + "host_pid": spec.get("hostPID") is True, + "host_ipc": spec.get("hostIPC") is True, + "host_path_volumes": host_paths, + "containers": containers, + } + + +def network_policy_record(raw: Dict[str, Any]) -> Dict[str, Any]: + meta, spec = raw.get("metadata") or {}, raw.get("spec") or {} + ingress, egress = spec.get("ingress") or [], spec.get("egress") or [] + # Unset policyTypes means Ingress, plus Egress when the policy has egress rules. + types = [_lower(t) for t in spec.get("policyTypes") or []] or (["ingress"] + (["egress"] if egress else [])) + everyone = selector_is_empty(spec.get("podSelector")) + return { + "namespace": meta.get("namespace"), + "name": meta.get("name"), + "pod_selector": spec.get("podSelector") or {}, + "selects_all_pods": everyone, + "policy_types": sorted(types), + "ingress_rules": len(ingress), + "egress_rules": len(egress), + # Every pod selected, the direction named, and no rule allowing anything. + "default_deny_ingress": everyone and "ingress" in types and not ingress, + "default_deny_egress": everyone and "egress" in types and not egress, + } + + +def workload_records(pods: List[Dict[str, Any]], policies: Optional[List[Dict[str, Any]]], + system: Iterable[str]) -> List[Dict[str, Any]]: + """Pods grouped by what runs them; the containers are those of the workload's least restricted pod template.""" + system = set(system) + by_namespace: Dict[str, List[Dict[str, Any]]] = {} + for policy in policies or []: + by_namespace.setdefault(str(policy["namespace"]), []).append(policy) + groups: Dict[Tuple[str, str, str], Dict[str, Any]] = {} + templates: Dict[Tuple[str, str, str], set] = {} + for pod in pods: + key = (str(pod["namespace"]), pod["workload_kind"], pod["workload_name"]) + selected = None if policies is None else any( + selector_matches(p["pod_selector"], pod["labels"]) for p in by_namespace.get(str(pod["namespace"]), [])) + containers = pod["containers"] + host_namespaces = pod["host_network"] or pod["host_pid"] or pod["host_ipc"] + # Linux security contexts do not apply to a Windows pod, so it is listed but never judged. + hardened = None if pod["windows"] else (bool(containers) and all(c["hardened"] for c in containers) + and not host_namespaces and not pod["host_path_volumes"]) + template = json.dumps([containers, pod["host_network"], pod["host_pid"], pod["host_ipc"], pod["host_path_volumes"], + pod["windows"]], sort_keys=True, default=str) + templates.setdefault(key, set()).add(template) + group = groups.get(key) + # During a rollout one workload runs two pod templates; the row shows the least restricted one, so a + # hardened old replica never hides what the new one runs. + if group is None or (hardened is False and group["hardened"] is not False): + group = groups[key] = { + "namespace": pod["namespace"], + "kind": pod["workload_kind"], + "name": pod["workload_name"], + "system_namespace": str(pod["namespace"]).lower() in system, + "windows": pod["windows"], + "pods": group["pods"] if group else 0, + "pod_templates": 0, + "host_network": pod["host_network"], + "host_pid": pod["host_pid"], + "host_ipc": pod["host_ipc"], + "host_path_volumes": pod["host_path_volumes"], + "uses_host_namespaces": host_namespaces, + "containers": containers, + "hardened": hardened, + "selected_by_network_policy": selected if group is None else group["selected_by_network_policy"], + } + group["pods"] += 1 + group["pod_templates"] = len(templates[key]) + # One replica outside every policy leaves the workload uncovered. + if selected is False: + group["selected_by_network_policy"] = False + return [groups[k] for k in sorted(groups)] + + +def namespace_records(namespaces: Optional[List[Dict[str, Any]]], workloads: Optional[List[Dict[str, Any]]], + policies: Optional[List[Dict[str, Any]]], system: Iterable[str]) -> List[Dict[str, Any]]: + """One record per namespace; the names come from the namespace list, else from the pods and policies read.""" + system = set(system) + labels: Dict[str, Optional[Dict[str, str]]] = {} + if namespaces is not None: + for ns in namespaces: + labels[str(dig(ns, "metadata", "name"))] = {str(k): str(v) for k, v in (dig(ns, "metadata", "labels") or {}).items()} + else: + for item in (workloads or []) + (policies or []): + labels.setdefault(str(item["namespace"]), None) + out = [] + for name in sorted(labels): + ns_labels = labels[name] + of_ns = [p for p in policies or [] if p["namespace"] == name] + mine = [w for w in workloads or [] if w["namespace"] == name] + + def psa(mode: str) -> Optional[str]: + return None if ns_labels is None else (ns_labels.get(f"{PSA_PREFIX}{mode}") or "none").lower() + + out.append({ + "name": name, + "system_namespace": name.lower() in system, + "pod_security_enforce": psa("enforce"), + "pod_security_audit": psa("audit"), + "pod_security_warn": psa("warn"), + "workloads": len(mine) if workloads is not None else None, + "pods": sum(w["pods"] for w in mine) if workloads is not None else None, + "network_policies": len(of_ns) if policies is not None else None, + "default_deny_ingress": any(p["default_deny_ingress"] for p in of_ns) if policies is not None else None, + "default_deny_egress": any(p["default_deny_egress"] for p in of_ns) if policies is not None else None, + }) + return out + + +# --- the Kubernetes API --- + +def parse_kubeconfig(value: str) -> Tuple[str, str]: + """(API server URL, CA certificate PEM) from a base64 kubeconfig; the user entry, which may hold a credential, is not read.""" + import yaml # lazy + + config = yaml.safe_load(base64.b64decode(value)) or {} + cluster = dig((config.get("clusters") or [{}])[0], "cluster") or {} + server, ca = cluster.get("server"), cluster.get("certificate-authority-data") + if not server or not ca: + raise ValueError("the cluster's kubeconfig names no API server or no certificate authority") + if not str(server).lower().startswith("https://"): + raise ValueError("the cluster's kubeconfig names an API server that is not https, so no token is sent to it") + return str(server).rstrip("/"), base64.b64decode(ca).decode() + + +def kube_list(client, server: str, ca_path: str, path: str) -> List[Dict[str, Any]]: + """Every page of one Kubernetes list.""" + items: List[Dict[str, Any]] = [] + params = {"limit": PAGE_SIZE} + while True: + response = client.get_url(f"{server}{path}", AKS_SERVER_APP, params, ca_path) + if response.status_code >= 400: + try: + message = str(response.json().get("message") or response.reason) + except ValueError: + message = str(response.reason or response.text[:200]) + raise KubeError(response.status_code, message) + body = response.json() + items.extend(body.get("items") or []) + token = dig(body, "metadata", "continue") + if not token: + return items + params = {"limit": PAGE_SIZE, "continue": token} + + +def cluster_identity(raw: Dict[str, Any]) -> Dict[str, Any]: + props = raw.get("properties") or {} + aad = props.get("aadProfile") or {} + engine = _lower(dig(props, "networkProfile", "networkPolicy")) or "none" + return { + "id": raw.get("id"), + "name": raw.get("name"), + "resource_group": resource_group_from_id(raw.get("id")), + "location": raw.get("location"), + "kubernetes_version": props.get("kubernetesVersion"), + "power_state": dig(props, "powerState", "code"), + "private_cluster": dig(props, "apiServerAccessProfile", "enablePrivateCluster") is True, + "entra_integrated": aad.get("managed") is True, + "azure_rbac_enabled": aad.get("enableAzureRBAC") is True, + "local_accounts_disabled": props.get("disableLocalAccounts") is True, + "network_plugin": dig(props, "networkProfile", "networkPlugin"), + "network_dataplane": dig(props, "networkProfile", "networkDataplane"), + # NetworkPolicy objects do nothing unless an engine enforces them. + "network_policy_engine": engine, + "network_policies_enforced": engine not in NO_POLICY_ENGINE, + "azure_policy_addon_enabled": dig(props, "addonProfiles", "azurepolicy", "enabled") is True, + } + + +def collect_cluster(client, raw: Dict[str, Any], settings: Dict[str, Any], collector: Collector) -> Dict[str, Any]: + """One cluster's record; a list that could not be read leaves its fields null and the failure recorded.""" + record: Dict[str, Any] = {**cluster_identity(raw), "kubernetes_api_read": False, "notes": [], + "namespaces_read": None, "pods_read": None, "network_policies_read": None, + "namespaces": None, "network_policies": None, "workloads": None, + "workloads_truncated": None, "pods_running": None, "pods_finished_skipped": None} + name = record["name"] + if _lower(record["power_state"]) == STOPPED: + # A stopped cluster has no API server and runs nothing, so there is nothing to read and nothing unread. + record.update({"kubernetes_api_read": True, "namespaces_read": True, "pods_read": True, "network_policies_read": True, + "namespaces": [], "network_policies": [], "workloads": [], "workloads_truncated": False, + "pods_running": 0, "pods_finished_skipped": 0}) + record["notes"].append("the cluster is stopped, so it runs no workload") + return record + if not record["entra_integrated"]: + collector.record(f"kubernetes({name})", PermissionError( + f"cluster {name} has no AKS-managed Microsoft Entra integration, so its Kubernetes API cannot be read with an Entra token")) + return record + + def _kubeconfig(): + body = client.post(f"{raw['id']}/listClusterUserCredential", AKS_API) + return parse_kubeconfig(((body.get("kubeconfigs") or [{}])[0]).get("value") or "") + + endpoint = collector.guard(f"containerservice.managedClusters.listClusterUserCredential({name})", _kubeconfig) + if endpoint is None: + return record + server, ca = endpoint + record["api_server"] = server + # The CA certificate is public material; requests reads a CA bundle from a file path. + with tempfile.NamedTemporaryFile("w", suffix=".pem") as ca_file: + ca_file.write(ca) + ca_file.flush() + + import requests # lazy + + unreachable = gave_up = False + + def _list(path: str, operation: str) -> Optional[List[Dict[str, Any]]]: + nonlocal unreachable, gave_up + if gave_up: + return None # one failure is recorded; repeating the same timeout or token error on every list adds nothing + try: + return kube_list(client, server, ca_file.name, path) + except KubeError as exc: # the API server answered and refused this list; the others may still be allowed + collector.record(f"kubernetes.{operation}.list({name})", exc) + except requests.RequestException as exc: # no answer: a private API server out of reach, DNS, TLS + unreachable = gave_up = True + collector.record(f"kubernetes.{operation}.list({name})", exc) + except Exception as exc: # noqa: BLE001 -- such as a token the tenant refused to issue for the cluster + gave_up = True + collector.record(f"kubernetes.{operation}.list({name})", exc) + return None + + raw_pods = _list("/api/v1/pods", "pods") + raw_policies = _list("/apis/networking.k8s.io/v1/networkpolicies", "networkpolicies") + raw_namespaces = _list("/api/v1/namespaces", "namespaces") + if unreachable: + record["notes"].append("the Kubernetes API server did not answer; a private cluster's API is reachable only from " + "its virtual network or a network peered with it") + + policies = sorted((network_policy_record(p) for p in raw_policies), key=lambda p: (str(p["namespace"]), str(p["name"]))) \ + if raw_policies is not None else None + workloads = None + if raw_pods is not None: + live = [p for p in raw_pods if _lower(dig(p, "status", "phase")) not in FINISHED_PHASES] + workloads = workload_records([pod_record(p) for p in live], policies, settings["system_namespaces"]) + record["pods_running"] = len(live) + record["pods_finished_skipped"] = len(raw_pods) - len(live) + record.update({ + "kubernetes_api_read": raw_pods is not None and raw_policies is not None and raw_namespaces is not None, + "namespaces_read": raw_namespaces is not None, + "pods_read": raw_pods is not None, + "network_policies_read": raw_policies is not None, + "namespaces": namespace_records(raw_namespaces, workloads, policies, settings["system_namespaces"]), + "network_policies": policies, + "workloads": workloads, + }) + if raw_namespaces is None and not gave_up: + record["notes"].append("the namespace list was not read, so namespaces come from the pods and policies read " + "and their Pod Security Admission labels are unknown") + if policies and not record["network_policies_enforced"]: + record["notes"].append("the cluster has NetworkPolicy objects but no network policy engine, so nothing enforces them") + return record + + +# --- summary --- + +def summarize(clusters: List[Dict[str, Any]], clusters_read: bool, settings: Dict[str, Any]) -> Dict[str, Any]: + pods_known = clusters_read and all(c["pods_read"] for c in clusters) + policies_known = clusters_read and all(c["network_policies_read"] for c in clusters) + namespaces_known = clusters_read and all(c["namespaces_read"] for c in clusters) + workloads = [w for c in clusters for w in c["workloads"] or []] + # Judged: Linux workloads outside the system namespaces. + judged = [w for w in workloads if not w["system_namespace"] and not w["windows"]] + containers = [k for w in judged for k in w["containers"]] + namespaces = [n for c in clusters for n in c["namespaces"] or [] if not n["system_namespace"]] + hardened = sum(1 for w in judged if w["hardened"]) + + def pod_count(value) -> Optional[int]: + return value if pods_known else None + + def policy_count(value) -> Optional[int]: + return value if policies_known else None + + def namespace_policy_count(value) -> Optional[int]: + # Without the namespace list a namespace holding no pod and no policy would be missed. + return value if (policies_known and namespaces_known) else None + + def containers_where(key: str) -> Optional[int]: + return pod_count(sum(1 for k in containers if k[key])) + + return { + "clusters_read": clusters_read, + "clusters": len(clusters), + "clusters_with_kubernetes_api_read": sum(1 for c in clusters if c["kubernetes_api_read"]), + "clusters_with_kubernetes_api_unread": sum(1 for c in clusters if not c["kubernetes_api_read"]), + "clusters_stopped": sum(1 for c in clusters if _lower(c["power_state"]) == STOPPED), + "clusters_without_entra_integration": sum(1 for c in clusters if not c["entra_integrated"]), + "clusters_enforcing_network_policies": sum(1 for c in clusters if c["network_policies_enforced"]), + "clusters_without_network_policy_engine": sum(1 for c in clusters if not c["network_policies_enforced"]), + "clusters_with_azure_policy_addon": sum(1 for c in clusters if c["azure_policy_addon_enabled"]), + "system_namespaces": list(settings["system_namespaces"]), + "namespaces": len(namespaces) if namespaces_known else None, + "namespaces_enforcing_pod_security": (sum(1 for n in namespaces if n["pod_security_enforce"] in PSA_ENFORCING) + if namespaces_known else None), + "namespaces_enforcing_restricted_pod_security": (sum(1 for n in namespaces if n["pod_security_enforce"] == "restricted") + if namespaces_known else None), + "namespaces_without_pod_security_enforcement": (sum(1 for n in namespaces if n["pod_security_enforce"] not in PSA_ENFORCING) + if namespaces_known else None), + "network_policies": policy_count(sum(len([p for p in c["network_policies"] or [] + if str(p["namespace"]).lower() not in settings["system_namespaces"]]) + for c in clusters)), + "namespaces_with_network_policy": namespace_policy_count(sum(1 for n in namespaces if n["network_policies"])), + "namespaces_without_network_policy": namespace_policy_count(sum(1 for n in namespaces if not n["network_policies"])), + "namespaces_with_default_deny_ingress": namespace_policy_count(sum(1 for n in namespaces if n["default_deny_ingress"])), + "namespaces_with_default_deny_egress": namespace_policy_count(sum(1 for n in namespaces if n["default_deny_egress"])), + "pods": pod_count(sum(c["pods_running"] or 0 for c in clusters)), + "pods_finished_skipped": pod_count(sum(c["pods_finished_skipped"] or 0 for c in clusters)), + "system_workloads": pod_count(sum(1 for w in workloads if w["system_namespace"])), + "windows_workloads": pod_count(sum(1 for w in workloads if w["windows"] and not w["system_namespace"])), + "workloads": pod_count(len(judged)), + "containers": pod_count(len(containers)), + "containers_non_root_enforced": containers_where("non_root_enforced"), + "containers_privilege_escalation_disabled": containers_where("privilege_escalation_disabled"), + "containers_read_only_root_filesystem": containers_where("read_only_root_filesystem"), + "containers_dropping_all_capabilities": containers_where("drops_all_capabilities"), + "containers_with_added_capabilities": containers_where("added_capabilities"), + "privileged_containers": containers_where("privileged"), + "containers_with_seccomp_profile": pod_count(sum(1 for k in containers if k["seccomp_profile"] in SECCOMP_CONFINED)), + "workloads_using_host_namespaces": pod_count(sum(1 for w in judged if w["uses_host_namespaces"])), + "workloads_with_host_path_volumes": pod_count(sum(1 for w in judged if w["host_path_volumes"])), + "workloads_hardened": pod_count(hardened), + "workloads_not_hardened": pod_count(len(judged) - hardened), + "workload_hardening_percentage": pod_count(coverage_percentage(hardened, len(judged))), + "workloads_selected_by_network_policy": (sum(1 for w in judged if w["selected_by_network_policy"]) + if (pods_known and policies_known) else None), + "workloads_without_network_policy": (sum(1 for w in judged if w["selected_by_network_policy"] is False) + if (pods_known and policies_known) else None), + "workload_rows_truncated": any(c["workloads_truncated"] for c in clusters), + } + + +# --- collection --- + +def collect(client, subscription_id: str, collector: Collector, exclusions: Optional[ResourceExclusions] = None, + settings: Optional[Dict[str, Any]] = None) -> tuple: + exclusions = exclusions if exclusions is not None else ResourceExclusions() + settings = settings if settings is not None else settings_from_env(collector) + + def _clusters(): + try: + return client.list(f"/subscriptions/{subscription_id}/providers/Microsoft.ContainerService/managedClusters", AKS_API) + except ArmError as exc: + if exc.code.lower() == "missingsubscriptionregistration": + return [] # a subscription that never registered the provider has no cluster + raise + + listed = collector.guard("containerservice.managedClusters.list", _clusters) + kept, excluded = exclusions.split(listed or [], "cluster") + exclusions.record_invalid(collector) + exclusions.warn_unmatched(logger) + + clusters = [] + for raw in sorted(kept, key=lambda c: arm_id_key(c.get("id"))): + record = collector.guard(f"collect({raw.get('name')})", lambda raw=raw: collect_cluster(client, raw, settings, collector)) + if record is None: + record = {**cluster_identity(raw), "kubernetes_api_read": False, "notes": [], "namespaces_read": None, + "pods_read": None, "network_policies_read": None, "namespaces": None, "network_policies": None, + "workloads": None, "workloads_truncated": None, "pods_running": None, "pods_finished_skipped": None} + clusters.append(record) + + summary = {**summarize(clusters, listed is not None, settings), + **exclusions.summary(excluded, ("cluster",), {"clusters": len(clusters) if listed is not None else None})} + # The counts above cover every workload; the rows are cut only afterwards, least hardened first kept. + for cluster in clusters: + rows = cluster["workloads"] + if rows is not None: + cluster["workloads_truncated"] = len(rows) > settings["max_rows"] + ordered = sorted(rows, key=lambda w: (w["system_namespace"], w["hardened"] is not False, + str(w["namespace"]), w["kind"], w["name"])) + cluster["workloads"] = ordered[: settings["max_rows"]] + summary["workload_rows_truncated"] = any(c["workloads_truncated"] for c in clusters) + results = {"clusters": clusters, "excluded": sorted(excluded, key=lambda e: arm_id_key(e.get("id")))} + return results, summary + + +def collect_subscription(client, subscription_id: str, collector: Collector) -> tuple: + return collect(client, subscription_id, collector, ResourceExclusions.from_env()) + + +def main() -> int: + return run_subscription(fetcher=NAME, logger=logger, collect=collect_subscription) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/fetchers/azure/aks_workload_security/fetcher.yaml b/fetchers/azure/aks_workload_security/fetcher.yaml new file mode 100644 index 00000000..73ddc9ee --- /dev/null +++ b/fetchers/azure/aks_workload_security/fetcher.yaml @@ -0,0 +1,143 @@ +name: azure_aks_workload_security +version: 0.1.0 +description: >- + Shows from each AKS cluster's Kubernetes API whether workload containers run non-root with privilege escalation off, a read-only root filesystem and capabilities dropped, and which namespaces network policies cover. +category: azure + +supports_targets: true + +runtime: + type: python + entry: fetcher.py + +output: + type: json + path: azure_aks_workload_security.json + aggregation: per_target + +target_schema: + subscription_id: + type: string + required: false + env: AZURE_SUBSCRIPTION_ID + description: > + Azure subscription id holding the AKS clusters. Optional - omit to use the first + enabled subscription the ambient credential can see. + environment: + type: string + required: false + env: AZURE_ENVIRONMENT + description: Deployment environment label (e.g. prod) written into the evidence metadata. + +config_schema: + system_namespaces: + type: string + required: false + env: AKS_SYSTEM_NAMESPACES + description: > + Namespaces that hold the platform's own pods, separated by commas or spaces. Their + workloads are listed and counted apart, and never judged, because node agents need + host access. Default: kube-system, kube-public, kube-node-lease, gatekeeper-system, + calico-system, tigera-operator, app-routing-system and aks-command. A value that + names no namespace fails the run. + max_rows: + type: integer + required: false + default: 500 + env: AKS_WORKLOAD_MAX_ROWS + description: > + Most workload rows kept per cluster, those not hardened first. Every count still + covers all workloads. + resource_exclusions: + type: patterns + prefixes: [rg, location, id] + bare: false + required: false + env: AZURE_RESOURCE_EXCLUSIONS + description: > + Resources outside the authorization boundary, to leave out of scope, + as case-insensitive glob patterns listed in the manifest, each with + a prefix: rg: (read from the resource id), + location: (the region's name without spaces, such as + usgovvirginia; a resource with no location never matches one) or + id: (also matches that resource's child resources). * + matches any run of characters and ? one character. A matching cluster is listed under excluded with the pattern, its Kubernetes API is not called, and it is left out of every count and judgement. A pattern that matches nothing is reported; a bare pattern fails the run. + +secrets: [] # DefaultAzureCredential only - see _categories/azure.yaml + +ksis: + - KSI-CNA-DFP # containers restricted to what they need: non-root, no privilege escalation, read-only root filesystem, capabilities dropped + - KSI-CNA-MAT # privileged containers, host namespaces and host paths that widen what a compromised pod can reach + - KSI-CNA-ULN # network policies and default-deny namespaces that limit traffic between workloads + +evidence_set: + reference_id: EVD-AZURE-AKS-WORKLOAD-SECURITY + name: Azure AKS Workload Security + instructions: | + Script: azure/aks_workload_security/fetcher.py + + Access: Reader on the subscription, plus on each cluster Azure Kubernetes Service Cluster User Role (to read the API server address and its CA certificate) and Azure Kubernetes Service RBAC Reader (to read pods, namespaces and network policies; it cannot read secrets). The cluster must use AKS-managed Microsoft Entra integration, and where it authorizes with Kubernetes RBAC instead of Azure RBAC, the identity needs a ClusterRole allowing get and list on those three resources in place of RBAC Reader. The run must reach its API server, which for a private cluster means running inside its virtual network or a peered one. Without these the cluster reads unread and the run fails. + + Commands Executed: + - subscriptions.list() — picks a subscription when none is set + - GET Microsoft.ContainerService/managedClusters — every AKS cluster with its power state, Entra integration, network policy engine and Azure Policy add-on + - POST managedClusters/{cluster}/listClusterUserCredential (per cluster) — the API server address and CA certificate from the user kubeconfig; the user entry is not read + - GET {API server}/api/v1/pods (per cluster) — every pod's security context, containers, host namespaces, host path volumes, labels and owner + - GET {API server}/apis/networking.k8s.io/v1/networkpolicies (per cluster) — every network policy with its pod selector, policy types and rule counts + - GET {API server}/api/v1/namespaces (per cluster) — every namespace with its Pod Security Admission labels + + Output: azure_aks_workload_security_.json — per cluster, its identity and network policy engine, each namespace with its Pod Security Admission levels and policy coverage, each network policy, and each workload (pods grouped by the Deployment, StatefulSet, DaemonSet or Job that runs them) with every container's security context; no environment variables, images, secrets or logs are read. + + Summary fields: + + Which clusters were read? + - clusters_read — True if the cluster list was read; false if it failed, and then every count below is 0 or null and the run fails + - clusters — Number of AKS clusters in scope + - clusters_with_kubernetes_api_read — Number of clusters whose pods, network policies and namespaces were all read; a stopped cluster counts, since it runs nothing + - clusters_with_kubernetes_api_unread — Number of clusters where a list could not be read: no AKS-managed Entra integration, a missing role, or an API server the run could not reach; the run fails + - clusters_stopped — Number of clusters that are stopped, which have no API server and run no workload + - clusters_without_entra_integration — Number of clusters without AKS-managed Microsoft Entra integration (none, or the legacy kind), whose Kubernetes API this fetcher cannot read + - clusters_enforcing_network_policies — Number of clusters with a network policy engine (azure, calico or cilium); only there do NetworkPolicy objects have any effect + - clusters_without_network_policy_engine — Number of clusters with no network policy engine, where every network policy below is unenforced + - clusters_with_azure_policy_addon — Number of clusters with the Azure Policy add-on, which can enforce pod security rules at admission + + Are namespaces restricted? + - system_namespaces — List of the namespaces treated as the platform's own (system_namespaces setting, or the default list); they are left out of every namespace, workload and container count below except system_workloads + - namespaces — Number of namespaces outside the system namespaces; null if any cluster's namespace list was not read + - namespaces_enforcing_pod_security — Number of those namespaces whose pod-security.kubernetes.io/enforce label is baseline or restricted, so the API server rejects pods that break that level; null if any namespace list was not read + - namespaces_enforcing_restricted_pod_security — Number of those namespaces enforcing the restricted level; null if any namespace list was not read + - namespaces_without_pod_security_enforcement — Number of those namespaces with no enforce label or the privileged level; another admission controller such as Azure Policy may still restrict them; null if any namespace list was not read + - network_policies — Number of network policies outside the system namespaces; null if any cluster's policies were not read + - namespaces_with_network_policy — Number of namespaces with at least one network policy; null if any cluster's policies or namespaces were not read + - namespaces_without_network_policy — Number of namespaces with no network policy, where every pod accepts and sends any traffic; null if any cluster's policies or namespaces were not read + - namespaces_with_default_deny_ingress — Number of namespaces with a policy that selects every pod, covers ingress and allows nothing, so only traffic another policy allows gets in; null if any cluster's policies or namespaces were not read + - namespaces_with_default_deny_egress — Number of namespaces with a policy that selects every pod, covers egress and allows nothing; null if any cluster's policies or namespaces were not read + + Are containers restricted? + - pods — Number of pods that are running or pending, in every namespace; null if any cluster's pods were not read + - pods_finished_skipped — Number of pods in the Succeeded or Failed phase, such as completed jobs, which are not assessed; null if any cluster's pods were not read + - system_workloads — Number of workloads in the system namespaces, listed but not judged; null if any cluster's pods were not read + - windows_workloads — Number of Windows workloads (by the pod's os field or node selector) outside the system namespaces, listed but not judged because Linux security contexts do not apply to them; null if any cluster's pods were not read + - workloads — Number of Linux workloads outside the system namespaces, each a set of pods run by one Deployment, StatefulSet, DaemonSet, Job or other controller, or a bare pod; null if any cluster's pods were not read + - containers — Number of containers and init containers in those workloads' pod template, counted once per workload, not per replica; a workload running two templates, as during a rollout, is represented by its least restricted one; null if any cluster's pods were not read + - containers_non_root_enforced — Number of containers with a non-zero runAsUser, or with no runAsUser and runAsNonRoot true, set on the container or its pod; a runAsUser of 0 is root and is not counted; null if any cluster's pods were not read + - containers_privilege_escalation_disabled — Number of containers with allowPrivilegeEscalation false; unset means allowed; null if any cluster's pods were not read + - containers_read_only_root_filesystem — Number of containers with readOnlyRootFilesystem true; null if any cluster's pods were not read + - containers_dropping_all_capabilities — Number of containers whose capabilities drop list includes ALL; null if any cluster's pods were not read + - containers_with_added_capabilities — Number of containers that add a Linux capability; null if any cluster's pods were not read + - privileged_containers — Number of containers with privileged true, which have the node's full access; null if any cluster's pods were not read + - containers_with_seccomp_profile — Number of containers whose seccomp profile, on the container or its pod, is RuntimeDefault or Localhost; null if any cluster's pods were not read + - workloads_using_host_namespaces — Number of workloads with hostNetwork, hostPID or hostIPC; null if any cluster's pods were not read + - workloads_with_host_path_volumes — Number of workloads that mount a path of the node's filesystem; null if any cluster's pods were not read + - workloads_hardened — Number of workloads where every container is non-root, has privilege escalation off, a read-only root filesystem, all capabilities dropped and none added, is not privileged, and the pod uses no host namespace or host path; null if any cluster's pods were not read + - workloads_not_hardened — Number of workloads that miss at least one of those; null if any cluster's pods were not read + - workload_hardening_percentage — Percentage of workloads that are hardened: workloads_hardened * 100 / workloads, 0 when there are none; null if any cluster's pods were not read + - workloads_selected_by_network_policy — Number of workloads whose every pod is selected by at least one network policy in its namespace; null if any cluster's pods or policies were not read + - workloads_without_network_policy — Number of workloads with a pod no network policy selects; null if any cluster's pods or policies were not read + - workload_rows_truncated — True if a cluster has more workloads than max_rows, so its workload list is cut (those not hardened are kept first) while every count above still covers them all + + What was left out of scope? + - resource_exclusion_patterns — List of the resource_exclusions patterns as given + - resource_exclusion_patterns_unmatched — List of the patterns that matched nothing, such as a mistyped name + - excluded_clusters — Number of clusters a pattern took out of scope; null if the cluster list was not read + - in_scope_clusters — Number of clusters assessed; null if the cluster list was not read diff --git a/tests/test_azure_aks_workload_security.py b/tests/test_azure_aks_workload_security.py new file mode 100644 index 00000000..5b28df04 --- /dev/null +++ b/tests/test_azure_aks_workload_security.py @@ -0,0 +1,558 @@ +"""azure_aks_workload_security: container security contexts, workload grouping, network policy coverage, Pod Security Admission labels, failure semantics. Synthetic payloads, no SDK, cluster or network.""" + +from __future__ import annotations + +import base64 +import importlib.util +import json +import logging +import sys +from collections import namedtuple +from pathlib import Path +from urllib.parse import urlparse + +import pytest +import requests +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent +AZURE = REPO_ROOT / "fetchers" / "azure" +sys.path.insert(0, str(REPO_ROOT / "fetchers" / "_lib")) +sys.path.insert(0, str(AZURE / "_shared")) + +_spec = importlib.util.spec_from_file_location("azure_aks_workload_security_under_test", + AZURE / "aks_workload_security" / "fetcher.py") +aks = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(aks) +azure_rest = sys.modules["azure_rest"] +azure_common = sys.modules["azure_common"] +Collector = azure_common.Collector +ResourceExclusions = sys.modules["resource_exclusions"].ResourceExclusions + +SUB = "/subscriptions/s1" +CLUSTER = f"{SUB}/resourceGroups/rg-aks/providers/Microsoft.ContainerService/managedClusters/aks-app" +OTHER = f"{SUB}/resourceGroups/rg-aks/providers/Microsoft.ContainerService/managedClusters/aks-old" +SERVER = "https://aks-app.hcp.usgovvirginia.cx.aks.containerservice.azure.us:443" +SETTINGS = {"system_namespaces": sorted(aks.DEFAULT_SYSTEM_NAMESPACES), "max_rows": 500} +HARDENED = {"runAsNonRoot": True, "allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, + "capabilities": {"drop": ["ALL"]}} + + +def raw_cluster(cid=CLUSTER, *, entra=True, engine="calico", power="Running", policy_addon=True): + props = {"kubernetesVersion": "1.31.2", "powerState": {"code": power}, + "apiServerAccessProfile": {"enablePrivateCluster": True}, + "networkProfile": {"networkPlugin": "azure", "networkDataplane": "azure"}, + "disableLocalAccounts": True, "addonProfiles": {"azurepolicy": {"enabled": policy_addon}}} + if engine: + props["networkProfile"]["networkPolicy"] = engine + if entra: + props["aadProfile"] = {"managed": True, "enableAzureRBAC": True} + return {"id": cid, "name": cid.rsplit("/", 1)[-1], "location": "usgovvirginia", "properties": props} + + +def container(name="app", **ctx): + return {"name": name, "image": "registry.example/app:1", "securityContext": ctx} if ctx else {"name": name} + + +def raw_pod(name, namespace="app", *, containers=None, init=None, pod_ctx=None, owner=("ReplicaSet", "web-7d9f"), + labels=None, phase="Running", **spec): + labels = labels if labels is not None else {"app": "web", "pod-template-hash": "7d9f"} + meta = {"name": name, "namespace": namespace, "labels": labels} + if owner: + meta["ownerReferences"] = [{"kind": owner[0], "name": owner[1], "controller": True}] + body = {"containers": containers if containers is not None else [container(**HARDENED)], **spec} + if init: + body["initContainers"] = init + if pod_ctx: + body["securityContext"] = pod_ctx + return {"metadata": meta, "spec": body, "status": {"phase": phase}} + + +def raw_policy(name, namespace="app", *, selector=None, types=None, ingress=None, egress=None): + spec = {"podSelector": selector or {}} + if types is not None: + spec["policyTypes"] = types + if ingress is not None: + spec["ingress"] = ingress + if egress is not None: + spec["egress"] = egress + return {"metadata": {"name": name, "namespace": namespace}, "spec": spec} + + +def raw_namespace(name, **labels): + return {"metadata": {"name": name, "labels": {f"pod-security.kubernetes.io/{k}": v for k, v in labels.items()}}} + + +def kubeconfig(server=SERVER, ca="-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n", user=True): + config = {"clusters": [{"name": "aks-app", "cluster": { + "server": server, "certificate-authority-data": base64.b64encode(ca.encode()).decode()}}]} + if user: + config["users"] = [{"name": "clusterUser", "user": {"token": "not-read"}}] + return base64.b64encode(yaml.safe_dump(config).encode()).decode() + + +# --- containers --- + +def test_a_hardened_container_and_what_each_missing_setting_costs() -> None: + full = aks.container_record(container(**HARDENED), {}, False) + assert full["hardened"] and full["non_root_enforced"] and full["drops_all_capabilities"] and not full["init_container"] + for missing in HARDENED: + ctx = {k: v for k, v in HARDENED.items() if k != missing} + assert aks.container_record(container(**ctx), {}, False)["hardened"] is False, missing + bare = aks.container_record(container(), {}, True) + assert bare["init_container"] and not bare["hardened"] and not bare["privilege_escalation_disabled"] + assert bare["seccomp_profile"] is None and bare["added_capabilities"] == [] + + +def test_explicit_false_and_weaker_values_are_not_read_as_safe() -> None: + loose = aks.container_record(container(runAsNonRoot=False, allowPrivilegeEscalation=True, readOnlyRootFilesystem=False, + capabilities={"drop": ["NET_RAW"], "add": ["net_admin"]}, privileged=True), {}, False) + assert not loose["non_root_enforced"] and not loose["privilege_escalation_disabled"] + assert not loose["read_only_root_filesystem"] and not loose["drops_all_capabilities"] + assert loose["privileged"] and loose["added_capabilities"] == ["NET_ADMIN"] and not loose["hardened"] + added = aks.container_record(container(**{**HARDENED, "capabilities": {"drop": ["all"], "add": ["NET_BIND_SERVICE"]}}), {}, False) + assert added["drops_all_capabilities"] and added["hardened"] is False + + +def test_pod_level_settings_apply_unless_the_container_overrides_them() -> None: + pod_ctx = {"runAsNonRoot": True, "runAsUser": 1000, "seccompProfile": {"type": "RuntimeDefault"}} + inherited = aks.container_record(container(allowPrivilegeEscalation=False), pod_ctx, False) + assert inherited["non_root_enforced"] and inherited["run_as_user"] == 1000 and inherited["seccomp_profile"] == "runtimedefault" + overridden = aks.container_record(container(runAsNonRoot=False, runAsUser=0), pod_ctx, False) + assert overridden["non_root_enforced"] is False and overridden["run_as_user"] == 0 + by_uid = aks.container_record(container(runAsUser=1001), {}, False) + assert by_uid["non_root_enforced"] and by_uid["run_as_non_root"] is False + root_uid = aks.container_record(container(runAsUser=0), {}, False) + assert root_uid["non_root_enforced"] is False + # allowPrivilegeEscalation and readOnlyRootFilesystem exist only on the container, never on the pod + pod_only = aks.container_record(container(), {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True}, False) + assert not pod_only["privilege_escalation_disabled"] and not pod_only["read_only_root_filesystem"] + + +# --- workloads --- + +def test_pods_are_grouped_by_what_runs_them() -> None: + assert aks.workload_of(raw_pod("web-7d9f-abc")) == ("Deployment", "web") + assert aks.workload_of(raw_pod("x", owner=("ReplicaSet", "web-7d9f"), labels={})) == ("ReplicaSet", "web-7d9f") + assert aks.workload_of(raw_pod("db-0", owner=("StatefulSet", "db"))) == ("StatefulSet", "db") + assert aks.workload_of(raw_pod("solo", owner=None)) == ("Pod", "solo") + pods = [aks.pod_record(raw_pod(f"web-7d9f-{i}")) for i in "abc"] + [aks.pod_record(raw_pod("db-0", owner=("StatefulSet", "db")))] + workloads = aks.workload_records(pods, [], SETTINGS["system_namespaces"]) + assert [(w["kind"], w["name"], w["pods"]) for w in workloads] == [("Deployment", "web", 3), ("StatefulSet", "db", 1)] + assert all(w["hardened"] and "labels" not in w for w in workloads) + + +def test_host_access_init_containers_and_windows_decide_hardening() -> None: + def one(**kwargs): + return aks.workload_records([aks.pod_record(raw_pod("p", **kwargs))], [], SETTINGS["system_namespaces"])[0] + assert one()["hardened"] is True + assert one(hostNetwork=True)["hardened"] is False and one(hostNetwork=True)["uses_host_namespaces"] + assert one(hostPID=True)["uses_host_namespaces"] and one(hostIPC=True)["uses_host_namespaces"] + mounted = one(volumes=[{"name": "h", "hostPath": {"path": "/var/run"}}, {"name": "e", "emptyDir": {}}]) + assert mounted["host_path_volumes"] == ["/var/run"] and mounted["hardened"] is False + assert one(init=[container("setup")])["hardened"] is False # an init container counts like any other + windows = one(os={"name": "windows"}) + assert windows["windows"] and windows["hardened"] is None + system = one(namespace="kube-system", hostNetwork=True) + assert system["system_namespace"] is True + assert one(containers=[])["hardened"] is False + + +# --- network policies --- + +@pytest.mark.parametrize("selector, labels, expected", [ + ({}, {"a": "1"}, True), (None, {}, True), + ({"matchLabels": {"app": "web"}}, {"app": "web", "x": "y"}, True), + ({"matchLabels": {"app": "web"}}, {"app": "api"}, False), + ({"matchLabels": {"app": "web", "tier": "fe"}}, {"app": "web"}, False), + ({"matchExpressions": [{"key": "app", "operator": "In", "values": ["web", "api"]}]}, {"app": "api"}, True), + ({"matchExpressions": [{"key": "app", "operator": "In", "values": ["web"]}]}, {}, False), + ({"matchExpressions": [{"key": "app", "operator": "NotIn", "values": ["web"]}]}, {"app": "web"}, False), + ({"matchExpressions": [{"key": "app", "operator": "NotIn", "values": ["web"]}]}, {}, True), + ({"matchExpressions": [{"key": "app", "operator": "Exists"}]}, {"app": ""}, True), + ({"matchExpressions": [{"key": "app", "operator": "Exists"}]}, {}, False), + ({"matchExpressions": [{"key": "app", "operator": "DoesNotExist"}]}, {"app": "x"}, False), + ({"matchExpressions": [{"key": "app", "operator": "Gt", "values": ["1"]}]}, {"app": "2"}, False), +]) +def test_label_selectors(selector, labels, expected) -> None: + assert aks.selector_matches(selector, labels) is expected + + +def test_default_deny_needs_every_pod_the_direction_and_no_rule() -> None: + deny = aks.network_policy_record(raw_policy("deny", types=["Ingress", "Egress"])) + assert deny["default_deny_ingress"] and deny["default_deny_egress"] and deny["selects_all_pods"] + implicit = aks.network_policy_record(raw_policy("implicit")) # no policyTypes: Ingress only + assert implicit["policy_types"] == ["ingress"] and implicit["default_deny_ingress"] and not implicit["default_deny_egress"] + scoped = aks.network_policy_record(raw_policy("scoped", selector={"matchLabels": {"app": "web"}}, types=["Ingress"])) + assert not scoped["default_deny_ingress"] and not scoped["selects_all_pods"] + allowing = aks.network_policy_record(raw_policy("allow", types=["Ingress"], ingress=[{"from": []}])) + assert not allowing["default_deny_ingress"] and allowing["ingress_rules"] == 1 + egress_rules = aks.network_policy_record(raw_policy("eg", egress=[{"to": []}])) + assert egress_rules["policy_types"] == ["egress", "ingress"] and not egress_rules["default_deny_egress"] + + +def test_a_workload_is_covered_only_when_every_pod_is_selected_in_its_own_namespace() -> None: + policies = [aks.network_policy_record(raw_policy("web", selector={"matchLabels": {"app": "web"}})), + aks.network_policy_record(raw_policy("all", namespace="other"))] + pods = [aks.pod_record(raw_pod("web-7d9f-a")), + aks.pod_record(raw_pod("web-7d9f-b", labels={"app": "canary", "pod-template-hash": "7d9f"})), + aks.pod_record(raw_pod("api-1", owner=("StatefulSet", "api"), labels={"app": "web"})), + aks.pod_record(raw_pod("job-1", owner=("Job", "job"), labels={"app": "job"}))] + by_name = {w["name"]: w for w in aks.workload_records(pods, policies, [])} + assert by_name["web"]["selected_by_network_policy"] is False # one replica is outside the policy + assert by_name["api"]["selected_by_network_policy"] is True + assert by_name["job"]["selected_by_network_policy"] is False # the other namespace's policy does not reach it + unread = aks.workload_records(pods, None, []) + assert all(w["selected_by_network_policy"] is None for w in unread) + + +def test_namespace_records_read_pod_security_labels_and_policy_coverage() -> None: + policies = [aks.network_policy_record(raw_policy("deny", types=["Ingress"]))] + workloads = aks.workload_records([aks.pod_record(raw_pod("web-7d9f-a"))], policies, SETTINGS["system_namespaces"]) + listed = [raw_namespace("app", enforce="Restricted", warn="baseline"), raw_namespace("empty"), raw_namespace("kube-system")] + by_name = {n["name"]: n for n in aks.namespace_records(listed, workloads, policies, SETTINGS["system_namespaces"])} + assert by_name["app"]["pod_security_enforce"] == "restricted" and by_name["app"]["pod_security_warn"] == "baseline" + assert by_name["app"]["pod_security_audit"] == "none" and by_name["app"]["default_deny_ingress"] is True + assert by_name["app"]["workloads"] == 1 and by_name["app"]["pods"] == 1 and by_name["app"]["network_policies"] == 1 + assert by_name["empty"]["network_policies"] == 0 and by_name["empty"]["default_deny_ingress"] is False + assert by_name["kube-system"]["system_namespace"] is True + derived = aks.namespace_records(None, workloads, policies, SETTINGS["system_namespaces"]) + assert [n["name"] for n in derived] == ["app"] and derived[0]["pod_security_enforce"] is None + + +def test_kubeconfig_gives_the_server_and_ca_and_rejects_one_without_them() -> None: + server, ca = aks.parse_kubeconfig(kubeconfig()) + assert server == SERVER and ca.startswith("-----BEGIN CERTIFICATE-----") + with pytest.raises(ValueError): + aks.parse_kubeconfig(base64.b64encode(b"clusters: []").decode()) + with pytest.raises(ValueError): + aks.parse_kubeconfig("") + + +def test_settings_from_env(monkeypatch) -> None: + monkeypatch.delenv(aks.SYSTEM_NAMESPACES_ENV, raising=False) + assert aks.settings_from_env()["system_namespaces"] == sorted(aks.DEFAULT_SYSTEM_NAMESPACES) + monkeypatch.setenv(aks.SYSTEM_NAMESPACES_ENV, " Kube-System, istio-system ") + assert aks.settings_from_env()["system_namespaces"] == ["istio-system", "kube-system"] + collector = Collector(logging.getLogger("test")) + monkeypatch.setenv(aks.SYSTEM_NAMESPACES_ENV, " , ") + aks.settings_from_env(collector) + assert not collector.ok and azure_common.classify_failure_code(collector.failures) == "bad_config" + + +# --- collection through a fake ARM and a fake Kubernetes API --- + +class _Resp: + def __init__(self, status, body, reason=""): + self.status_code, self._body, self.reason = status, body, reason + self.text = json.dumps(body) if not isinstance(body, str) else body + self.headers: dict = {} + + def json(self): + if isinstance(self._body, str): + raise ValueError("not json") + return self._body + + +PODS = [ + raw_pod("web-7d9f-a"), raw_pod("web-7d9f-b"), + raw_pod("api-0", owner=("StatefulSet", "api"), labels={"app": "api"}, containers=[container(runAsNonRoot=True)]), + raw_pod("agent-x", namespace="kube-system", owner=("DaemonSet", "agent"), labels={"k8s-app": "agent"}, + containers=[container(privileged=True)], hostNetwork=True), + raw_pod("migrate-1", owner=("Job", "migrate"), labels={"job": "migrate"}, phase="Succeeded", containers=[container()]), + raw_pod("batch-1", namespace="batch", owner=("Job", "batch"), labels={"job": "batch"}, containers=[container(privileged=True)]), +] +POLICIES = [raw_policy("deny-all", types=["Ingress", "Egress"]), + raw_policy("web", selector={"matchLabels": {"app": "web"}}, types=["Ingress"], ingress=[{"from": []}]), + raw_policy("sys", namespace="kube-system")] +NAMESPACES = [raw_namespace("app", enforce="restricted"), raw_namespace("batch"), raw_namespace("kube-system"), + raw_namespace("idle", enforce="baseline")] + + +class _Api: + """ARM and one cluster's Kubernetes API behind one fake session.""" + + def __init__(self, *, clusters=None, list_fail=False, credential_status=200, forbid=(), unreachable=False, + registered=True, page=None): + self.clusters = clusters if clusters is not None else [raw_cluster()] + self.list_fail, self.credential_status, self.forbid = list_fail, credential_status, set(forbid) + self.unreachable, self.registered, self.page = unreachable, registered, page + self.calls: list[tuple] = [] + + def post(self, url, params=None, headers=None, timeout=None): + path = urlparse(url).path + self.calls.append(("POST", path, headers["Authorization"], None)) + if self.credential_status != 200: + return _Resp(self.credential_status, {"error": {"code": "AuthorizationFailed", "message": "the action is not allowed"}}) + return _Resp(200, {"kubeconfigs": [{"name": "clusterUser", "value": kubeconfig()}]}) + + def get(self, url, params=None, headers=None, timeout=None, verify=None): + parsed = urlparse(url) + self.calls.append(("GET", parsed.path, headers["Authorization"], verify)) + if parsed.netloc.startswith("aks-app.hcp"): + return self._kube(parsed.path, params or {}, verify) + if parsed.path.endswith("/providers/Microsoft.ContainerService/managedClusters"): + if not self.registered: + return _Resp(409, {"error": {"code": "MissingSubscriptionRegistration", "message": "not registered"}}) + if self.list_fail: + return _Resp(403, {"error": {"code": "AuthorizationFailed", "message": "no"}}) + return _Resp(200, {"value": self.clusters}) + return _Resp(404, {"error": {"code": "NotFound", "message": parsed.path}}) + + def _kube(self, path, params, verify): + if self.unreachable: + raise requests.ConnectionError("Failed to establish a new connection: name or service not known") + assert verify and Path(verify).read_text().startswith("-----BEGIN CERTIFICATE-----") + kind = path.rsplit("/", 1)[-1] + if kind in self.forbid: + return _Resp(403, {"kind": "Status", "message": f"{kind} is forbidden: User cannot list resource", "code": 403}) + items = {"pods": PODS, "networkpolicies": POLICIES, "namespaces": NAMESPACES}[kind] + if self.page and kind == "pods": + start = int(params.get("continue") or 0) + chunk = items[start:start + self.page] + more = start + self.page < len(items) + return _Resp(200, {"items": chunk, "metadata": {"continue": str(start + self.page)} if more else {}}) + return _Resp(200, {"items": items, "metadata": {}}) + + +class _Cred: + def __init__(self): + self.scopes: list[str] = [] + + def get_token(self, scope): + self.scopes.append(scope) + return namedtuple("T", "token expires_on")(f"tok:{scope}", 9_999_999_999) + + +def _collect(api, exclusions=None, settings=None, cred=None): + client = azure_rest.AzureRestClient(cred or _Cred(), session=api) + collector = Collector(logging.getLogger("test")) + results, summary = aks.collect(client, "s1", collector, exclusions, settings or SETTINGS) + return results, summary, collector + + +def test_collect_reads_a_cluster_and_judges_only_workload_namespaces(assert_summary_documented, monkeypatch) -> None: + monkeypatch.setattr(azure_rest.time, "sleep", lambda s: None) + api, cred = _Api(), _Cred() + results, summary, collector = _collect(api, cred=cred) + assert collector.ok, collector.failures + assert_summary_documented("azure/aks_workload_security", summary) + cluster = results["clusters"][0] + assert cluster["kubernetes_api_read"] and cluster["api_server"] == SERVER and cluster["network_policy_engine"] == "calico" + assert cluster["pods_running"] == 5 and cluster["pods_finished_skipped"] == 1 + assert summary["clusters_with_kubernetes_api_read"] == 1 and summary["clusters_enforcing_network_policies"] == 1 + assert summary["pods"] == 5 and summary["pods_finished_skipped"] == 1 + assert summary["system_workloads"] == 1 and summary["workloads"] == 3 and summary["containers"] == 3 + assert summary["workloads_hardened"] == 1 and summary["workloads_not_hardened"] == 2 + assert summary["workload_hardening_percentage"] == 33 + assert summary["containers_non_root_enforced"] == 2 and summary["privileged_containers"] == 1 # kube-system's is not counted + assert summary["workloads_using_host_namespaces"] == 0 + assert summary["namespaces"] == 3 and summary["namespaces_enforcing_pod_security"] == 2 + assert summary["namespaces_enforcing_restricted_pod_security"] == 1 and summary["namespaces_without_pod_security_enforcement"] == 1 + assert summary["network_policies"] == 2 and summary["namespaces_with_network_policy"] == 1 + assert summary["namespaces_without_network_policy"] == 2 + assert summary["namespaces_with_default_deny_ingress"] == 1 and summary["namespaces_with_default_deny_egress"] == 1 + assert summary["workloads_selected_by_network_policy"] == 2 and summary["workloads_without_network_policy"] == 1 + # least hardened first, system namespaces last + assert [(w["namespace"], w["name"]) for w in cluster["workloads"]] == [ + ("app", "api"), ("batch", "batch"), ("app", "web"), ("kube-system", "agent")] + # ARM is called with an ARM token, the cluster with a token for the AKS server application and the cluster's CA + kube = [c for c in api.calls if c[1].startswith(("/api/", "/apis/"))] + assert len(kube) == 3 and all(c[2] == f"Bearer tok:{aks.AKS_SERVER_APP}/.default" and c[3] for c in kube) + assert not Path(kube[0][3]).exists() # the CA file is gone once the cluster is read + arm = [c for c in api.calls if not c[1].startswith(("/api/", "/apis/"))] + assert all(aks.AKS_SERVER_APP not in c[2] for c in arm) + assert "not-read" not in json.dumps(results) + + +def test_pods_are_paged(monkeypatch) -> None: + _, summary, collector = _collect(_Api(page=2)) + assert collector.ok and summary["pods"] == 5 and summary["workloads"] == 3 + + +def test_a_forbidden_list_nulls_only_what_depends_on_it_and_fails() -> None: + results, summary, collector = _collect(_Api(forbid={"pods"})) + assert not collector.ok and azure_common.classify_failure_code(collector.failures) == "not_authorized" + assert results["clusters"][0]["workloads"] is None and results["clusters"][0]["kubernetes_api_read"] is False + assert summary["workloads"] is None and summary["workloads_hardened"] is None and summary["pods"] is None + assert summary["workload_hardening_percentage"] is None and summary["workloads_without_network_policy"] is None + assert summary["network_policies"] == 2 and summary["namespaces"] == 3 and summary["clusters_with_kubernetes_api_unread"] == 1 + + results, summary, collector = _collect(_Api(forbid={"networkpolicies"})) + assert not collector.ok and summary["network_policies"] is None and summary["namespaces_without_network_policy"] is None + assert summary["workloads"] == 3 and summary["workloads_selected_by_network_policy"] is None + assert all(w["selected_by_network_policy"] is None for w in results["clusters"][0]["workloads"]) + + results, summary, collector = _collect(_Api(forbid={"namespaces"})) + assert not collector.ok and summary["namespaces"] is None and summary["namespaces_enforcing_pod_security"] is None + assert summary["namespaces_without_network_policy"] is None and summary["workloads"] == 3 + assert {n["name"] for n in results["clusters"][0]["namespaces"]} == {"app", "batch", "kube-system"} + assert any("namespace list was not read" in note for note in results["clusters"][0]["notes"]) + + +def test_an_unreachable_private_api_server_is_tried_once_and_fails_as_unreachable(monkeypatch) -> None: + monkeypatch.setattr(azure_rest.time, "sleep", lambda s: None) + api = _Api(unreachable=True) + results, summary, collector = _collect(api) + assert len(collector.failures) == 1 and azure_common.classify_failure_code(collector.failures) == "target_unreachable" + assert {c[1] for c in api.calls if c[1].startswith(("/api/", "/apis/"))} == {"/api/v1/pods"} + cluster = results["clusters"][0] + assert cluster["kubernetes_api_read"] is False and any("did not answer" in n for n in cluster["notes"]) + assert summary["workloads"] is None and summary["network_policies"] is None and summary["namespaces"] is None + + +def test_a_refused_credential_call_and_a_cluster_without_entra_leave_the_cluster_unread() -> None: + results, summary, collector = _collect(_Api(credential_status=403)) + assert not collector.ok and azure_common.classify_failure_code(collector.failures) == "not_authorized" + assert results["clusters"][0]["kubernetes_api_read"] is False and summary["workloads"] is None + + api = _Api(clusters=[raw_cluster(entra=False)]) + results, summary, collector = _collect(api) + assert not collector.ok and "no AKS-managed Microsoft Entra integration" in collector.failures[0]["message"] + assert summary["clusters_without_entra_integration"] == 1 and summary["workloads"] is None + assert not [c for c in api.calls if c[0] == "POST"] # no credential is requested for a cluster it cannot read + + +def test_a_stopped_cluster_reads_as_running_nothing() -> None: + api = _Api(clusters=[raw_cluster(power="Stopped")]) + results, summary, collector = _collect(api) + assert collector.ok and summary["clusters_stopped"] == 1 and summary["workloads"] == 0 and summary["namespaces"] == 0 + assert summary["workload_hardening_percentage"] == 0 and summary["clusters_with_kubernetes_api_read"] == 1 + assert len(api.calls) == 1 + + +def test_no_policy_engine_is_counted_and_noted() -> None: + results, summary, collector = _collect(_Api(clusters=[raw_cluster(engine=None, policy_addon=False)])) + assert collector.ok and summary["clusters_without_network_policy_engine"] == 1 + assert summary["clusters_enforcing_network_policies"] == 0 and summary["clusters_with_azure_policy_addon"] == 0 + assert any("no network policy engine" in n for n in results["clusters"][0]["notes"]) + + +def test_failed_cluster_list_an_unregistered_provider_and_exclusions() -> None: + _, summary, collector = _collect(_Api(list_fail=True)) + assert not collector.ok and summary["clusters_read"] is False and summary["workloads"] is None + assert summary["in_scope_clusters"] is None and summary["excluded_clusters"] is None + + _, summary, collector = _collect(_Api(registered=False)) + assert collector.ok and summary["clusters"] == 0 and summary["workloads"] == 0 + + api = _Api(clusters=[raw_cluster(), raw_cluster(OTHER, entra=False)]) + results, summary, collector = _collect(api, ResourceExclusions(["id:*/aks-old"])) + assert collector.ok and summary["clusters"] == 1 and summary["excluded_clusters"] == 1 + assert [e["name"] for e in results["excluded"]] == ["aks-old"] + + _, _, collector = _collect(_Api(), ResourceExclusions(["aks-old"])) + assert not collector.ok and azure_common.classify_failure_code(collector.failures) == "bad_config" + + +def test_row_cap_keeps_the_unhardened_workloads_and_every_count() -> None: + results, summary, collector = _collect(_Api(), settings={**SETTINGS, "max_rows": 2}) + assert collector.ok and summary["workload_rows_truncated"] is True and summary["workloads"] == 3 + assert [w["name"] for w in results["clusters"][0]["workloads"]] == ["api", "batch"] + assert results["clusters"][0]["workloads_truncated"] is True + + +def test_main_writes_evidence(monkeypatch, tmp_path) -> None: + monkeypatch.setattr(azure_rest, "pinned_credential", lambda sub: _Cred()) + monkeypatch.setattr(requests, "Session", lambda: _Api()) + monkeypatch.setenv("AZURE_SUBSCRIPTION_ID", "s1") + monkeypatch.setenv("EVIDENCE_DIR", str(tmp_path)) + monkeypatch.delenv(aks.SYSTEM_NAMESPACES_ENV, raising=False) + monkeypatch.delenv("AZURE_RESOURCE_EXCLUSIONS", raising=False) + assert aks.main() == 0 + evidence = json.loads((tmp_path / "azure_aks_workload_security_s1.json").read_text()) + assert evidence["metadata"]["partial_failure"] is False and evidence["summary"]["workloads"] == 3 + + +def test_a_refused_cluster_token_is_recorded_once_and_not_called_unreachable() -> None: + class Refusing(_Cred): + def get_token(self, scope): + if scope.startswith(aks.AKS_SERVER_APP): + raise RuntimeError("AADSTS53003: access blocked by Conditional Access") + return super().get_token(scope) + + results, summary, collector = _collect(_Api(), cred=Refusing()) + assert len(collector.failures) == 1 and azure_common.classify_failure_code(collector.failures) == "auth_failed" + assert results["clusters"][0]["notes"] == [] and summary["workloads"] is None + + +def test_a_rollout_shows_the_least_restricted_pod_template_whichever_pod_comes_first() -> None: + old = raw_pod("web-aaa-1", owner=("ReplicaSet", "web-aaa"), labels={"app": "web", "pod-template-hash": "aaa"}) + new = raw_pod("web-bbb-1", owner=("ReplicaSet", "web-bbb"), labels={"app": "web", "pod-template-hash": "bbb"}, + containers=[container(privileged=True)], hostNetwork=True) + for order in ([old, new], [new, old], [old, new, old]): + workload, = aks.workload_records([aks.pod_record(p) for p in order], [], SETTINGS["system_namespaces"]) + assert workload["name"] == "web" and workload["pods"] == len(order) and workload["pod_templates"] == 2 + assert workload["hardened"] is False and workload["uses_host_namespaces"] is True + assert workload["containers"][0]["privileged"] is True + same, = aks.workload_records([aks.pod_record(old), aks.pod_record(old)], [], SETTINGS["system_namespaces"]) + assert same["pod_templates"] == 1 and same["hardened"] is True + + +def test_a_root_uid_is_root_whatever_run_as_non_root_says() -> None: + assert aks.container_record(container(runAsNonRoot=True, runAsUser=0), {}, False)["non_root_enforced"] is False + assert aks.container_record(container(runAsNonRoot=True), {"runAsUser": 0}, False)["non_root_enforced"] is False + assert aks.container_record(container(runAsUser=True), {}, False)["non_root_enforced"] is False # not a uid + assert aks.container_record(container(runAsUser=65534), {"runAsNonRoot": False}, False)["non_root_enforced"] is True + + +@pytest.mark.parametrize("spec", [{"os": {"name": "Windows"}}, {"nodeSelector": {"kubernetes.io/os": "windows"}}, + {"nodeSelector": {"beta.kubernetes.io/os": "windows"}}]) +def test_a_windows_pod_is_recognised_by_os_or_node_selector(spec) -> None: + workload, = aks.workload_records([aks.pod_record(raw_pod("w", containers=[container()], **spec))], [], []) + assert workload["windows"] is True and workload["hardened"] is None + linux, = aks.workload_records([aks.pod_record(raw_pod("l", nodeSelector={"kubernetes.io/os": "linux"}))], [], []) + assert linux["windows"] is False and linux["hardened"] is True + + +def test_no_token_is_sent_to_an_api_server_that_is_not_https() -> None: + with pytest.raises(ValueError, match="not https"): + aks.parse_kubeconfig(kubeconfig(server="http://aks-app.hcp.example")) + + class Plain(_Api): + def post(self, url, params=None, headers=None, timeout=None): + return _Resp(200, {"kubeconfigs": [{"name": "clusterUser", "value": kubeconfig(server="http://aks-app.hcp.example")}]}) + + api = Plain() + _, summary, collector = _collect(api) + assert not collector.ok and summary["workloads"] is None + assert not [c for c in api.calls if c[1].startswith(("/api/", "/apis/"))] + + +def test_a_privileged_container_is_never_hardened_and_an_egress_only_policy_denies_no_ingress() -> None: + assert aks.container_record(container(**HARDENED, privileged=True), {}, False)["hardened"] is False + egress_only = aks.network_policy_record(raw_policy("eg", types=["Egress"])) + assert egress_only["default_deny_egress"] is True and egress_only["default_deny_ingress"] is False + + +def _summary(pods, namespaces=(), policies=()): + workloads = aks.workload_records([aks.pod_record(p) for p in pods], [aks.network_policy_record(p) for p in policies], + SETTINGS["system_namespaces"]) + cluster = {**aks.cluster_identity(raw_cluster()), "kubernetes_api_read": True, "namespaces_read": True, "pods_read": True, + "network_policies_read": True, "workloads": workloads, "workloads_truncated": False, + "network_policies": [aks.network_policy_record(p) for p in policies], "pods_running": len(pods), + "pods_finished_skipped": 0, + "namespaces": aks.namespace_records(list(namespaces), workloads, [], SETTINGS["system_namespaces"])} + return aks.summarize([cluster], True, SETTINGS) + + +def test_windows_workloads_unconfined_seccomp_and_the_privileged_psa_level_are_not_counted_as_restricted() -> None: + summary = _summary( + [raw_pod("win-1", owner=("DaemonSet", "win"), containers=[container(privileged=True)], os={"name": "windows"}), + raw_pod("a-1", owner=("StatefulSet", "a"), containers=[container(**HARDENED, seccompProfile={"type": "Unconfined"})]), + raw_pod("b-1", owner=("StatefulSet", "b"), containers=[container(**HARDENED, seccompProfile={"type": "RuntimeDefault"})]), + raw_pod("c-1", owner=("StatefulSet", "c"), pod_ctx={"seccompProfile": {"type": "Localhost"}})], + [raw_namespace("app", enforce="privileged"), raw_namespace("strict", enforce="baseline"), raw_namespace("open")]) + assert summary["windows_workloads"] == 1 and summary["workloads"] == 3 and summary["containers"] == 3 + assert summary["privileged_containers"] == 0 and summary["workloads_hardened"] == 3 # the Windows pod is not judged + assert summary["containers_with_seccomp_profile"] == 2 + assert summary["namespaces"] == 3 and summary["namespaces_enforcing_pod_security"] == 1 + assert summary["namespaces_without_pod_security_enforcement"] == 2 + + +def test_legacy_or_unmanaged_entra_profiles_are_not_entra_integrated() -> None: + legacy = raw_cluster() + legacy["properties"]["aadProfile"] = {"managed": False, "clientAppID": "x"} + assert aks.cluster_identity(legacy)["entra_integrated"] is False + assert aks.cluster_identity(raw_cluster())["entra_integrated"] is True + assert aks.cluster_identity(raw_cluster(entra=False))["azure_rbac_enabled"] is False diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 181753c9..282bf18c 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -229,6 +229,7 @@ def test_instructions_open_with_the_entry_path() -> None: "azure/activity_log_alerts", "azure/ai_hub_configuration", "azure/aks_cluster_configuration", + "azure/aks_workload_security", "azure/alert_rules", "azure/api_management_configuration", "azure/app_gateway_backend_tls", From 08f50e2685b5b4ae15991d47dc227b0306609b70 Mon Sep 17 00:00:00 2001 From: Soya Kawamura Date: Tue, 17 Nov 2026 13:43:15 -0700 Subject: [PATCH 2/2] aks_workload_security: an API server that does not answer is tried once Review follow-up: - requests.ConnectTimeout is a ConnectionError, so AzureRestClient._send retried an unreachable Kubernetes API server 5 times at the full 60 s timeout plus 15 s of backoff, about 315 s per cluster; two such clusters passed the runner's 600 s limit and lost every cluster's evidence. _send takes retry_connect (default True); get_url passes False and a 10 s connect timeout. 429 and 5xx answers are still retried. - The unreachable test compared a set of paths, so 5 requests read as one. It now checks the list of requests and that nothing slept. New tests go through a real AzureRestClient with a session that raises ConnectTimeout or ConnectionError, and check a 503 is still retried. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MBcaFn4pMcbUNLVCX6bo1j --- fetchers/azure/_shared/azure_rest.py | 21 ++++++++--- tests/test_azure_aks_workload_security.py | 46 ++++++++++++++++++++++- 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/fetchers/azure/_shared/azure_rest.py b/fetchers/azure/_shared/azure_rest.py index db289895..9d6178dd 100644 --- a/fetchers/azure/_shared/azure_rest.py +++ b/fetchers/azure/_shared/azure_rest.py @@ -33,6 +33,8 @@ LAW_API = "2023-09-01" DIAG_API = "2021-05-01-preview" HTTP_TIMEOUT = 60 +# Seconds to open a connection to a server outside ARM that may be out of reach (get_url). +CONNECT_TIMEOUT = 10 QUERY_TIMEOUT = 600 # Throttling and transient server errors, retried as azure-core's policy does for the SDK fetchers. RETRY_STATUSES = frozenset({408, 429, 500, 502, 503, 504}) @@ -138,8 +140,12 @@ def _validate_read_url(self, url: str, resource: str, method: str) -> bool: allow_subscription_root=method == "get") return True - def _send(self, method: str, url: str, resource: str, headers: Optional[Dict[str, str]] = None, **kwargs: Any): - """One request, retried on throttling, transient 5xx and dropped connections, and a GET on a timeout.""" + def _send(self, method: str, url: str, resource: str, headers: Optional[Dict[str, str]] = None, + retry_connect: bool = True, **kwargs: Any): + """One request, retried on throttling, transient 5xx and dropped connections, and a GET on a timeout. + + retry_connect False raises on the first connection failure (a connect timeout included): for a server that + may be out of reach, where every retry waits out the whole timeout again. 429 and 5xx are still retried.""" import requests # lazy guarded = self._validate_read_url(url, resource, method) @@ -151,7 +157,7 @@ def _send(self, method: str, url: str, resource: str, headers: Optional[Dict[str try: response = getattr(self.session, method)(url, headers=sent, **kwargs) except requests.ConnectionError: - if attempt == MAX_RETRIES: + if not retry_connect or attempt == MAX_RETRIES: raise time.sleep(float(min(2 ** attempt, MAX_RETRY_WAIT))) continue @@ -199,11 +205,14 @@ def _follow(self, body: Dict[str, Any]) -> List[Dict[str, Any]]: def get_url(self, url: str, resource: str, params: Optional[Dict[str, str]] = None, verify: Any = None): """A GET to a service outside ARM that takes an Entra token for `resource`, such as an AKS cluster's Kubernetes - API; `verify` is the CA bundle path for a server whose certificate a private CA signed. The raw response.""" - kwargs: Dict[str, Any] = {"params": params, "timeout": HTTP_TIMEOUT} + API; `verify` is the CA bundle path for a server whose certificate a private CA signed. The raw response. + + Such a server may be out of reach (a private cluster, or authorized IP ranges that leave out the runner), so a + connection failure is not retried and the connect timeout is short; 429 and 5xx answers are still retried.""" + kwargs: Dict[str, Any] = {"params": params, "timeout": (CONNECT_TIMEOUT, HTTP_TIMEOUT)} if verify is not None: kwargs["verify"] = verify - return self._send("get", url, resource, **kwargs) + return self._send("get", url, resource, retry_connect=False, **kwargs) def post(self, path: str, api_version: str) -> Dict[str, Any]: """An ARM read action that takes no body, such as fetchEventDetails.""" diff --git a/tests/test_azure_aks_workload_security.py b/tests/test_azure_aks_workload_security.py index 5b28df04..5fff0dcb 100644 --- a/tests/test_azure_aks_workload_security.py +++ b/tests/test_azure_aks_workload_security.py @@ -391,11 +391,13 @@ def test_a_forbidden_list_nulls_only_what_depends_on_it_and_fails() -> None: def test_an_unreachable_private_api_server_is_tried_once_and_fails_as_unreachable(monkeypatch) -> None: - monkeypatch.setattr(azure_rest.time, "sleep", lambda s: None) + sleeps: list = [] + monkeypatch.setattr(azure_rest.time, "sleep", sleeps.append) api = _Api(unreachable=True) results, summary, collector = _collect(api) assert len(collector.failures) == 1 and azure_common.classify_failure_code(collector.failures) == "target_unreachable" - assert {c[1] for c in api.calls if c[1].startswith(("/api/", "/apis/"))} == {"/api/v1/pods"} + # One request in all: no retry of the pods list, and the other lists are not tried. + assert [c[1] for c in api.calls if c[1].startswith(("/api/", "/apis/"))] == ["/api/v1/pods"] and sleeps == [] cluster = results["clusters"][0] assert cluster["kubernetes_api_read"] is False and any("did not answer" in n for n in cluster["notes"]) assert summary["workloads"] is None and summary["network_policies"] is None and summary["namespaces"] is None @@ -556,3 +558,43 @@ def test_legacy_or_unmanaged_entra_profiles_are_not_entra_integrated() -> None: assert aks.cluster_identity(legacy)["entra_integrated"] is False assert aks.cluster_identity(raw_cluster())["entra_integrated"] is True assert aks.cluster_identity(raw_cluster(entra=False))["azure_rbac_enabled"] is False + + +class _Session: + """A requests.Session stand-in for one Kubernetes API server: raises `error`, or answers `statuses` in turn.""" + + def __init__(self, error=None, statuses=()): + self.error, self.statuses, self.calls = error, list(statuses), [] + + def get(self, url, params=None, headers=None, timeout=None, verify=None, **kwargs): + self.calls.append(timeout) + if self.error is not None: + raise self.error + status = self.statuses.pop(0) + return _Resp(status, {"items": [], "metadata": {}} if status == 200 else {"message": "busy"}) + + +@pytest.mark.parametrize("error", [requests.ConnectTimeout("connect timed out"), + requests.ConnectionError("Failed to establish a new connection")]) +def test_a_connection_failure_to_the_api_server_is_not_retried(monkeypatch, error) -> None: + # Through the real AzureRestClient: each retry of a dropped connection would wait out the whole timeout again. + sleeps: list = [] + monkeypatch.setattr(azure_rest.time, "sleep", sleeps.append) + session = _Session(error=error) + client = azure_rest.AzureRestClient(_Cred(), session=session) + collector = Collector(logging.getLogger("test")) + with pytest.raises(requests.ConnectionError): + aks.kube_list(client, SERVER, None, "/api/v1/pods") + assert len(session.calls) == 1 and sleeps == [] + assert session.calls[0] == (azure_rest.CONNECT_TIMEOUT, azure_rest.HTTP_TIMEOUT) + collector.record("kubernetes.pods.list(c)", error) + assert azure_common.classify_failure_code(collector.failures) == "target_unreachable" + + +def test_a_busy_api_server_is_still_retried(monkeypatch) -> None: + # The server answered a 503, so a retry can help. + monkeypatch.setattr(azure_rest.time, "sleep", lambda s: None) + session = _Session(statuses=[503, 200]) + client = azure_rest.AzureRestClient(_Cred(), session=session) + assert aks.kube_list(client, SERVER, None, "/api/v1/pods") == [] + assert len(session.calls) == 2