diff --git a/hack/aks/Makefile b/hack/aks/Makefile index c8973236c0..36058aa42d 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -27,6 +27,8 @@ IP_PREFIX ?= serviceTaggedIp PUBLIC_IP_ID ?= /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/publicIPAddresses PUBLIC_IPv4 ?= $(PUBLIC_IP_ID)/$(IP_PREFIX)-$(CLUSTER)-v4 PUBLIC_IPv6 ?= $(PUBLIC_IP_ID)/$(IP_PREFIX)-$(CLUSTER)-v6 +NAT_GATEWAY_V2 ?= ponv6-nat-gateway-v2 +NAT_GATEWAY_V2_ID ?= /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/natGateways/$(NAT_GATEWAY_V2) KUBE_PROXY_JSON_PATH ?= ./kube-proxy.json LTS ?= auto ACNS ?= false @@ -171,6 +173,42 @@ vnetscale-swift-net-up: ## Create vnet, nodenet and podnet subnets for vnet scal $(AZCLI) network vnet subnet create -g $(GROUP) --vnet-name $(VNET) --name nodenet --address-prefixes 10.240.0.0/16 -o none $(AZCLI) network vnet subnet create -g $(GROUP) --vnet-name $(VNET) --name podnet --address-prefixes 10.40.0.0/13 -o none +vnetscale-dualstack-swift-net-up: rg-up ## Create dual-stack vnet, nodenet and podnet subnets for vnet scale + @$(AZCLI) network vnet show -g $(GROUP) --name $(VNET) -o none 2>/dev/null || \ + $(AZCLI) network vnet create -g $(GROUP) -l $(REGION) --name $(VNET) \ + --address-prefixes 10.0.0.0/8 fd00:aec6:6946::/48 -o none + $(AZCLI) network vnet update -g $(GROUP) --name $(VNET) \ + --address-prefixes 10.0.0.0/8 fd00:aec6:6946::/48 -o none + $(AZCLI) network vnet subnet create -g $(GROUP) --vnet-name $(VNET) --name nodenet \ + --address-prefixes 10.240.0.0/16 fd00:aec6:6946::/64 -o none + $(AZCLI) network vnet subnet create -g $(GROUP) --vnet-name $(VNET) --name podnet \ + --address-prefixes 10.241.0.0/16 fd00:aec6:6946:1::/64 -o none + +nat-gateway-v2-up: vnetscale-dualstack-swift-net-up ## Create a dual-stack StandardV2 NAT Gateway and attach it to both PONv6 subnets + $(AZCLI) network public-ip create -g $(GROUP) -l $(REGION) \ + --name $(IP_PREFIX)-$(CLUSTER)-v4 --allocation-method Static --sku StandardV2 \ + --tier Regional --version IPv4 --ip-tags $(IP_TAG) -o none + $(AZCLI) network public-ip create -g $(GROUP) -l $(REGION) \ + --name $(IP_PREFIX)-$(CLUSTER)-v6 --allocation-method Static --sku StandardV2 \ + --tier Regional --version IPv6 --ip-tags $(IP_TAG) -o none + $(AZCLI) rest --method put \ + --url "https://management.azure.com$(NAT_GATEWAY_V2_ID)?api-version=2025-05-01" \ + --body '{"location":"$(REGION)","sku":{"name":"StandardV2"},"properties":{"publicIpAddresses":[{"id":"$(PUBLIC_IPv4)"}],"publicIpAddressesV6":[{"id":"$(PUBLIC_IPv6)"}]}}' \ + -o none + @for attempt in $$(seq 1 60); do \ + state=$$($(AZCLI) rest --method get \ + --url "https://management.azure.com$(NAT_GATEWAY_V2_ID)?api-version=2025-05-01" \ + --query properties.provisioningState -o tsv); \ + if [ "$$state" = "Succeeded" ]; then break; fi; \ + if [ "$$state" = "Failed" ]; then echo "NAT Gateway V2 provisioning failed"; exit 1; fi; \ + if [ "$$attempt" = "60" ]; then echo "Timed out waiting for NAT Gateway V2"; exit 1; fi; \ + sleep 10; \ + done + $(AZCLI) network vnet subnet update -g $(GROUP) --vnet-name $(VNET) --name nodenet \ + --nat-gateway $(NAT_GATEWAY_V2_ID) -o none + $(AZCLI) network vnet subnet update -g $(GROUP) --vnet-name $(VNET) --name podnet \ + --nat-gateway $(NAT_GATEWAY_V2_ID) -o none + overlay-net-up: ## Create vnet, nodenet subnets @$(AZCLI) network vnet show -g $(GROUP) --name $(VNET) -o none 2>/dev/null || \ $(AZCLI) network vnet create -g $(GROUP) -l $(REGION) --name $(VNET) --address-prefixes 10.0.0.0/8 -o none @@ -364,6 +402,19 @@ vnetscale-swift-byocni-nokubeproxy-up: rg-up ipv4 vnetscale-swift-net-up ## Brin --yes @$(MAKE) set-kubeconf +vnetscale-dualstack-swift-byocni-nokubeproxy-up: nat-gateway-v2-up ## Bring up a dual-stack Vnet Scale SWIFT BYO CNI cluster without kube-proxy + $(COMMON_AKS_FIELDS) \ + --outbound-type userAssignedNATGateway \ + --network-plugin none \ + --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ + --pod-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/podnet \ + --kube-proxy-config $(KUBE_PROXY_JSON_PATH) \ + --pod-ip-allocation-mode StaticBlock \ + --ip-families ipv4,ipv6 \ + --aks-custom-headers AKSHTTPCustomFeatures=Microsoft.ContainerService/AzureVnetScaleDualStackPreview,AKSHTTPCustomFeatures=Microsoft.ContainerService/Ubuntu2404Preview \ + --yes + @$(MAKE) set-kubeconf + vnetscale-swift-cilium-up: rg-up ipv4 vnetscale-swift-net-up ## Bring up a Vnet Scale SWIFT Cilium cluster $(COMMON_AKS_FIELDS) \ --load-balancer-outbound-ips $(PUBLIC_IPv4) \ diff --git a/hack/aks/deploy.mk b/hack/aks/deploy.mk index 2259939ced..f23078712f 100644 --- a/hack/aks/deploy.mk +++ b/hack/aks/deploy.mk @@ -171,3 +171,13 @@ deploy-ebpf-podsubnet-cilium: print-ebpf-cilium-vars deploy-common-ebpf-cilium ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/ebpf/podsubnet/cilium.yaml \ | kubectl apply -f - @$(MAKE) wait-for-cilium + +deploy-ebpf-podsubnet-dualstack-cilium: print-ebpf-cilium-vars deploy-common-ebpf-cilium + @kubectl apply -f ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/ebpf/podsubnet-dualstack/static/ + CILIUM_VERSION_TAG=$(EBPF_CILIUM_VERSION_TAG) CILIUM_IMAGE_REGISTRY=$(EBPF_CILIUM_IMAGE_REGISTRY) \ + envsubst '$${CILIUM_VERSION_TAG},$${CILIUM_IMAGE_REGISTRY},$${IPV6_HP_BPF_VERSION},$${IPV6_IMAGE_REGISTRY},$${AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY},$${AZURE_IPTABLES_MONITOR_TAG}' < \ + ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/ebpf/podsubnet-dualstack/cilium.yaml \ + | kubectl apply -f - + @$(MAKE) wait-for-cilium + @test "$$(kubectl get daemonset cilium -n kube-system -o jsonpath='{.spec.template.spec.volumes[?(@.name=="azure-ip-masq-dir")].configMap.name}')" = "azure-dns-imds-ip-masq-agent-config" + @! kubectl get daemonset cilium -n kube-system -o jsonpath='{.spec.template.spec.containers[*].name}' | grep -qw azure-ip-masq-merger diff --git a/test/integration/manifests/cilium/v1.17/ebpf/podsubnet-dualstack/cilium.yaml b/test/integration/manifests/cilium/v1.17/ebpf/podsubnet-dualstack/cilium.yaml new file mode 100644 index 0000000000..31d62b6cd9 --- /dev/null +++ b/test/integration/manifests/cilium/v1.17/ebpf/podsubnet-dualstack/cilium.yaml @@ -0,0 +1,533 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + annotations: + meta.helm.sh/release-name: cilium + meta.helm.sh/release-namespace: kube-system + labels: + app.kubernetes.io/actually-managed-by: Eno + app.kubernetes.io/managed-by: Helm + k8s-app: cilium + kubernetes.azure.com/managedby: aks + name: cilium + namespace: kube-system +spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: cilium + template: + metadata: + annotations: + prometheus.io/port: "9962" + prometheus.io/scrape: "true" + labels: + app.kubernetes.io/managed-by: Eno + k8s-app: cilium + kubernetes.azure.com/ebpf-dataplane: cilium + kubernetes.azure.com/managedby: aks + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: kubernetes.io/os + operator: In + values: + - linux + automountServiceAccountToken: true + containers: + - args: + - --config-dir=/tmp/cilium/config-map + command: + - cilium-agent + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: KUBE_CLIENT_BACKOFF_BASE + value: "1" + - name: KUBE_CLIENT_BACKOFF_DURATION + value: "120" + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 10 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + - name: require-k8s-connectivity + value: "false" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + name: cilium-agent + ports: + - containerPort: 9962 + hostPort: 9962 + name: prometheus + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + resources: {} + securityContext: + appArmorProfile: + type: Unconfined + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + startupProbe: + failureThreshold: 105 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + - mountPath: /sys/fs/bpf + mountPropagation: HostToContainer + name: bpf-maps + - mountPath: /var/run/cilium + name: cilium-run + - mountPath: /host/etc/cni/net.d + name: etc-cni-netd + - mountPath: /var/lib/cilium/clustermesh + name: clustermesh-secrets + readOnly: true + - mountPath: /tmp/cilium/config-map + name: cilium-config-path + readOnly: true + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + - mountPath: /var/run/cilium/netns + mountPropagation: HostToContainer + name: cilium-netns + - mountPath: /flowlog-config + name: hubble-flowlog-config + readOnly: true + - mountPath: /var/log/acns/hubble + name: networkflowlogs + - mountPath: /etc/config + name: azure-ip-masq-dir + readOnly: true + - command: + - /azure-iptables-monitor + - -v + - "3" + - -events=true + - -checkMap=true + - -ipv6=true + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: $AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY/azure-iptables-monitor:$AZURE_IPTABLES_MONITOR_TAG + imagePullPolicy: IfNotPresent + name: azure-iptables-monitor + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/config + name: iptables-config + - mountPath: /etc/config6 + name: ip6tables-config + - mountPath: /azure-block-iptables-bpf-map + name: iptables-block-bpf-map + readOnly: true + dnsPolicy: ClusterFirst + hostNetwork: true + initContainers: + - command: + - /azure-block-iptables + - -mode=attach + - -overwrite=true + image: $AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY/azure-iptables-monitor:$AZURE_IPTABLES_MONITOR_TAG + imagePullPolicy: IfNotPresent + name: iptables-blocker-init + resources: {} + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /sys/fs/bpf + name: bpf-maps + - mountPath: /proc + name: hostproc + - command: + - /azure-iptables-monitor + - -v + - "3" + - -events=true + - -terminateOnSuccess=true + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: $AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY/azure-iptables-monitor:$AZURE_IPTABLES_MONITOR_TAG + imagePullPolicy: IfNotPresent + name: azure-iptables-monitor-init + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/config + name: iptables-config + - command: + - /install-plugin.sh + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + name: install-cni-binaries + resources: {} + securityContext: + capabilities: + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-path + - command: + - sh + - -ec + - | + cp /usr/bin/cilium-mount /hostbin/cilium-mount; + nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; + rm /hostbin/cilium-mount + env: + - name: CGROUP_ROOT + value: /run/cilium/cgroupv2 + - name: BIN_PATH + value: /opt/cni/bin + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + name: mount-cgroup + resources: {} + securityContext: + appArmorProfile: + type: Unconfined + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /hostproc + name: hostproc + - mountPath: /hostbin + name: cni-path + - command: + - sh + - -ec + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + env: + - name: BIN_PATH + value: /opt/cni/bin + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + name: apply-sysctl-overwrites + resources: {} + securityContext: + appArmorProfile: + type: Unconfined + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /hostproc + name: hostproc + - mountPath: /hostbin + name: cni-path + - args: + - mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf + command: + - /bin/bash + - -c + - -- + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + name: mount-bpf-fs + resources: {} + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + name: bpf-maps + - command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + key: clean-cilium-state + name: cilium-config + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + key: clean-cilium-bpf-state + name: cilium-config + optional: true + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + name: clean-cilium-state + resources: + requests: + cpu: 100m + memory: 100Mi + securityContext: + appArmorProfile: + type: Unconfined + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /sys/fs/bpf + name: bpf-maps + - mountPath: /run/cilium/cgroupv2 + mountPropagation: HostToContainer + name: cilium-cgroup + - mountPath: /var/run/cilium + name: cilium-run + - command: + - /ipv6-hp-bpf + image: $IPV6_IMAGE_REGISTRY/ipv6-hp-bpf:$IPV6_HP_BPF_VERSION + imagePullPolicy: IfNotPresent + name: start-ipv6-hp-bpf + resources: {} + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/log + name: ipv6-hp-bpf + priorityClassName: system-node-critical + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccount: cilium + serviceAccountName: cilium + terminationGracePeriodSeconds: 1 + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + volumes: + - hostPath: + path: /var/log + type: DirectoryOrCreate + name: ipv6-hp-bpf + - hostPath: + path: /etc/systemd + type: DirectoryOrCreate + name: host-etc-systemd + - hostPath: + path: /lib/systemd + type: DirectoryOrCreate + name: host-lib-systemd + - hostPath: + path: /usr/lib + type: DirectoryOrCreate + name: host-usr-lib + - hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + name: cilium-run + - hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + name: bpf-maps + - hostPath: + path: /proc + type: Directory + name: hostproc + - hostPath: + path: /run/cilium/cgroupv2 + type: DirectoryOrCreate + name: cilium-cgroup + - hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + name: cni-path + - hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + name: etc-cni-netd + - hostPath: + path: /lib/modules + type: "" + name: lib-modules + - hostPath: + path: /run/xtables.lock + type: FileOrCreate + name: xtables-lock + - name: clustermesh-secrets + secret: + defaultMode: 256 + optional: true + secretName: cilium-clustermesh + - configMap: + defaultMode: 420 + name: cilium-config + name: cilium-config-path + - hostPath: + path: /proc/sys/net + type: Directory + name: host-proc-sys-net + - hostPath: + path: /proc/sys/kernel + type: Directory + name: host-proc-sys-kernel + - hostPath: + path: /var/run/netns + type: DirectoryOrCreate + name: cilium-netns + - configMap: + defaultMode: 420 + name: acns-flowlog-config + optional: true + name: hubble-flowlog-config + - hostPath: + path: /var/log/acns/hubble + type: DirectoryOrCreate + name: networkflowlogs + - configMap: + defaultMode: 420 + name: allowed-iptables-patterns + optional: true + name: iptables-config + - configMap: + defaultMode: 420 + name: allowed-ip6tables-patterns + optional: true + name: ip6tables-config + - hostPath: + path: /sys/fs/bpf/azure-block-iptables + type: DirectoryOrCreate + name: iptables-block-bpf-map + - configMap: + defaultMode: 420 + name: azure-dns-imds-ip-masq-agent-config + optional: true + name: azure-ip-masq-dir + updateStrategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 5% + type: RollingUpdate diff --git a/test/integration/manifests/cilium/v1.17/ebpf/podsubnet-dualstack/static/allowed-ip6tables-patterns.yaml b/test/integration/manifests/cilium/v1.17/ebpf/podsubnet-dualstack/static/allowed-ip6tables-patterns.yaml new file mode 100644 index 0000000000..b463118e2a --- /dev/null +++ b/test/integration/manifests/cilium/v1.17/ebpf/podsubnet-dualstack/static/allowed-ip6tables-patterns.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: allowed-ip6tables-patterns + namespace: kube-system +data: + filter: | + -A INPUT -p udp --dport 68 -j ACCEPT + + global: | + ^-N .* + ^-P .* + ^-A (KUBE-FIREWALL) .* + + ^-A .* -j (KUBE-FIREWALL) + ^-A IP-MASQ-AGENT + ^-A .* -j IP-MASQ-AGENT + + ^.*--comment.*cilium: + ^.*--comment.*cilium-feeder: + ^.*--comment.*AKS managed: added by AgentBaker + ^-A CILIUM_\S+ + mangle: | + + nat: | + -A POSTROUTING -j SWIFT + -A SWIFT -s + -A POSTROUTING -j SWIFT-POSTROUTING + -A SWIFT-POSTROUTING -s + raw: | + ^.*--comment.*localdns: skip conntrack + security: | diff --git a/test/integration/manifests/cilium/v1.17/ebpf/podsubnet-dualstack/static/azure-dns-imds-ip-masq-agent-config.yaml b/test/integration/manifests/cilium/v1.17/ebpf/podsubnet-dualstack/static/azure-dns-imds-ip-masq-agent-config.yaml new file mode 100644 index 0000000000..cfbf2fa1f0 --- /dev/null +++ b/test/integration/manifests/cilium/v1.17/ebpf/podsubnet-dualstack/static/azure-dns-imds-ip-masq-agent-config.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +data: + ip-masq-agent: "nonMasqueradeCIDRs:\n- 0.0.0.0/1\n- 128.0.0.0/3\n- 160.0.0.0/5\n- + 168.0.0.0/11\n- 168.32.0.0/12\n- 168.48.0.0/13\n- 168.56.0.0/14\n- 168.60.0.0/15\n- + 168.62.0.0/16\n- 168.63.0.0/17\n- 168.63.128.0/24\n- 168.63.129.0/29\n- 168.63.129.8/30\n- + 168.63.129.12/30\n- 168.63.129.17/32\n- 168.63.129.18/31\n- 168.63.129.20/30\n- + 168.63.129.24/29\n- 168.63.129.32/27\n- 168.63.129.64/26\n- 168.63.129.128/25\n- + 168.63.130.0/23\n- 168.63.132.0/22\n- 168.63.136.0/21\n- 168.63.144.0/20\n- 168.63.160.0/19\n- + 168.63.192.0/18\n- 168.64.0.0/10\n- 168.128.0.0/9\n- 169.0.0.0/9\n- 169.128.0.0/10\n- + 169.192.0.0/11\n- 169.224.0.0/12\n- 169.240.0.0/13\n- 169.248.0.0/14\n- 169.252.0.0/15\n- + 169.254.0.0/17\n- 169.254.128.0/19\n- 169.254.160.0/21\n- 169.254.168.0/24\n- + 169.254.169.0/25\n- 169.254.169.128/26\n- 169.254.169.192/27\n- 169.254.169.224/28\n- + 169.254.169.240/29\n- 169.254.169.248/30\n- 169.254.169.252/31\n- 169.254.169.255/32\n- + 169.254.170.0/23\n- 169.254.172.0/22\n- 169.254.176.0/20\n- 169.254.192.0/18\n- + 169.255.0.0/16\n- 170.0.0.0/7\n- 172.0.0.0/6\n- 176.0.0.0/4\n- 192.0.0.0/3\n- + 224.0.0.0/3\n- \"::/0\"\n \nmasqLinkLocal: true\nmasqLinkLocalIPv6: true\n" +kind: ConfigMap +metadata: + annotations: + meta.helm.sh/release-name: cilium + meta.helm.sh/release-namespace: kube-system + labels: + app.kubernetes.io/actually-managed-by: Eno + app.kubernetes.io/managed-by: Helm + name: azure-dns-imds-ip-masq-agent-config + namespace: kube-system diff --git a/test/integration/manifests/cilium/v1.17/ebpf/podsubnet-dualstack/static/cilium-config.yaml b/test/integration/manifests/cilium/v1.17/ebpf/podsubnet-dualstack/static/cilium-config.yaml new file mode 100644 index 0000000000..6dcd2d5dff --- /dev/null +++ b/test/integration/manifests/cilium/v1.17/ebpf/podsubnet-dualstack/static/cilium-config.yaml @@ -0,0 +1,167 @@ +apiVersion: v1 +data: + agent-not-ready-taint-key: node.cilium.io/agent-not-ready + arping-refresh-period: 30s + auto-direct-node-routes: "false" + bpf-algorithm-annotation: "false" + bpf-events-drop-enabled: "true" + bpf-events-policy-verdict-enabled: "true" + bpf-events-trace-enabled: "true" + bpf-lb-acceleration: disabled + bpf-lb-external-clusterip: "false" + bpf-lb-map-max: "65536" + bpf-lb-mode: snat + bpf-lb-mode-annotation: "false" + bpf-lb-sock: "false" + bpf-lb-sock-hostns-only: "true" + bpf-lb-sock-terminate-pod-connections: "false" + bpf-lb-source-range-all-types: "false" + bpf-map-dynamic-size-ratio: "0.0025" + bpf-policy-map-max: "16384" + bpf-root: /sys/fs/bpf + ces-slice-mode: fcfs + cgroup-root: /run/cilium/cgroupv2 + cilium-endpoint-gc-interval: 5m0s + cluster-id: "0" + cluster-name: alewpodsubebpfcilcanary + cni-exclusive: "false" + cni-log-file: /var/run/cilium/cilium-cni.log + datapath-mode: veth + debug: "false" + direct-routing-skip-unreachable: "false" + disable-cnp-status-updates: "true" + disable-embedded-dns-proxy: "false" + disable-endpoint-crd: "false" + dnsproxy-enable-transparent-mode: "false" + egress-gateway-reconciliation-trigger-interval: 1s + enable-auto-protect-node-port-range: "true" + enable-bgp-control-plane: "false" + enable-bpf-clock-probe: "true" + enable-bpf-masquerade: "true" + enable-cilium-endpoint-slice: "true" + enable-endpoint-health-checking: "false" + enable-endpoint-lockdown-on-policy-overflow: "false" + enable-endpoint-routes: "true" + enable-experimental-lb: "false" + enable-health-check-loadbalancer-ip: "false" + enable-health-check-nodeport: "true" + enable-health-checking: "true" + enable-host-legacy-routing: "false" + enable-hubble: "false" + enable-hubble-open-metrics: "false" + enable-internal-traffic-policy: "true" + enable-ip-masq-agent: "true" + enable-ipv4: "true" + enable-ipv4-big-tcp: "false" + enable-ipv4-masquerade: "true" + enable-ipv6: "true" + enable-ipv6-big-tcp: "false" + enable-ipv6-masquerade: "true" + enable-k8s-networkpolicy: "true" + enable-k8s-terminating-endpoint: "true" + enable-l2-neigh-discovery: "true" + enable-l7-proxy: "true" + enable-lb-ipam: "false" + enable-local-node-route: "false" + enable-local-redirect-policy: "true" + enable-masquerade-to-route-source: "false" + enable-metrics: "true" + enable-node-selector-labels: "false" + enable-non-default-deny-policies: "true" + enable-policy: default + enable-remote-node-identity: "true" + enable-remote-node-masquerade: "false" + enable-runtime-device-detection: "false" + enable-sctp: "false" + enable-session-affinity: "true" + enable-source-ip-verification: "false" + enable-standalone-dns-proxy: "true" + enable-svc-source-range-check: "true" + enable-tcx: "false" + enable-vtep: "false" + enable-well-known-identities: "false" + enable-wireguard: "false" + enable-xt-socket-fallback: "true" + external-envoy-proxy: "false" + health-check-icmp-failure-threshold: "3" + hubble-disable-tls: "false" + hubble-event-buffer-capacity: "4095" + hubble-export-file-max-backups: "5" + hubble-export-file-max-size-mb: "10" + hubble-flowlogs-config-path: /flowlog-config/flowlogs.yaml + hubble-listen-address: :4244 + hubble-metrics: flow:sourceEgressContext=pod;destinationIngressContext=pod tcp:sourceEgressContext=pod;destinationIngressContext=pod + drop:sourceEgressContext=pod;destinationIngressContext=pod dns:sourceEgressContext=pod;destinationIngressContext=pod + hubble-metrics-server: :9965 + hubble-socket-path: /var/run/cilium/hubble.sock + hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt + hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt + hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key + identity-allocation-mode: crd + install-iptables-rules: "true" + install-no-conntrack-iptables-rules: "false" + ipam: delegated-plugin + ipam-cilium-node-update-rate: 15s + ipv4-native-routing-cidr: 10.241.0.0/16 + ipv6-native-routing-cidr: fd00:aec6:6946:1::/64 + k8s-client-burst: "20" + k8s-client-qps: "10" + k8s-require-ipv4-pod-cidr: "false" + k8s-require-ipv6-pod-cidr: "false" + kube-proxy-replacement: "true" + kube-proxy-replacement-healthz-bind-address: 0.0.0.0:10256 + local-router-ipv4: 169.254.23.0 + mesh-auth-enabled: "false" + mesh-auth-gc-interval: 5m0s + mesh-auth-queue-size: "1024" + mesh-auth-rotated-identities-queue-size: "1024" + metrics: +cilium_bpf_map_pressure +cilium_proxy_datapath_update_timeout_total + monitor-aggregation: medium + monitor-aggregation-flags: all + monitor-aggregation-interval: 5s + nat-map-stats-entries: "32" + nat-map-stats-interval: 30s + node-port-bind-protection: "true" + nodeport-addresses: "" + nodes-gc-interval: 5m0s + operator-api-serve-addr: 127.0.0.1:9234 + operator-prometheus-serve-addr: :9963 + preallocate-bpf-maps: "false" + procfs: /host/proc + prometheus-serve-addr: :9962 + proxy-connect-timeout: "2" + proxy-idle-timeout-seconds: "60" + proxy-max-connection-duration-seconds: "0" + proxy-max-requests-per-connection: "0" + proxy-xff-num-trusted-hops-egress: "0" + proxy-xff-num-trusted-hops-ingress: "0" + remove-cilium-node-taints: "true" + routing-mode: native + set-cilium-is-up-condition: "true" + set-cilium-node-taints: "true" + sidecar-istio-proxy-image: cilium/istio_proxy + synchronize-k8s-nodes: "true" + tofqdns-dns-reject-response-code: refused + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "1000" + tofqdns-idle-connection-grace-period: 0s + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-min-ttl: "3600" + tofqdns-proxy-port: "40046" + tofqdns-proxy-response-max-delay: 100ms + tofqdns-server-port: "40045" + unmanaged-pod-watcher-interval: "0" + vtep-cidr: "" + vtep-endpoint: "" + vtep-mac: "" + vtep-mask: "" +kind: ConfigMap +metadata: + annotations: + meta.helm.sh/release-name: cilium + meta.helm.sh/release-namespace: kube-system + labels: + app.kubernetes.io/actually-managed-by: Eno + app.kubernetes.io/managed-by: Helm + name: cilium-config + namespace: kube-system diff --git a/test/integration/manifests/cilium/v1.18/ebpf/podsubnet-dualstack/cilium.yaml b/test/integration/manifests/cilium/v1.18/ebpf/podsubnet-dualstack/cilium.yaml new file mode 100644 index 0000000000..349f894dd3 --- /dev/null +++ b/test/integration/manifests/cilium/v1.18/ebpf/podsubnet-dualstack/cilium.yaml @@ -0,0 +1,537 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + annotations: + meta.helm.sh/release-name: cilium + meta.helm.sh/release-namespace: kube-system + labels: + app.kubernetes.io/actually-managed-by: Eno + app.kubernetes.io/managed-by: Helm + k8s-app: cilium + kubernetes.azure.com/managedby: aks + name: cilium + namespace: kube-system +spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: cilium + template: + metadata: + annotations: + prometheus.io/port: "9962" + prometheus.io/scrape: "true" + labels: + app.kubernetes.io/managed-by: Eno + k8s-app: cilium + kubernetes.azure.com/ebpf-dataplane: cilium + kubernetes.azure.com/managedby: aks + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: kubernetes.io/os + operator: In + values: + - linux + automountServiceAccountToken: true + containers: + - args: + - --config-dir=/tmp/cilium/config-map + command: + - cilium-agent + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: KUBE_CLIENT_BACKOFF_BASE + value: "1" + - name: KUBE_CLIENT_BACKOFF_DURATION + value: "120" + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium-distroless:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 10 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + - name: require-k8s-connectivity + value: "false" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + name: cilium-agent + ports: + - containerPort: 9962 + hostPort: 9962 + name: prometheus + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + resources: {} + securityContext: + appArmorProfile: + type: Unconfined + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + startupProbe: + failureThreshold: 105 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + - mountPath: /sys/fs/bpf + mountPropagation: HostToContainer + name: bpf-maps + - mountPath: /var/run/cilium + name: cilium-run + - mountPath: /host/etc/cni/net.d + name: etc-cni-netd + - mountPath: /var/lib/cilium/clustermesh + name: clustermesh-secrets + readOnly: true + - mountPath: /tmp/cilium/config-map + name: cilium-config-path + readOnly: true + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + - mountPath: /var/run/cilium/netns + mountPropagation: HostToContainer + name: cilium-netns + - mountPath: /flowlog-config + name: hubble-flowlog-config + readOnly: true + - mountPath: /var/log/acns/hubble + name: networkflowlogs + - mountPath: /etc/config + name: azure-ip-masq-dir + readOnly: true + - command: + - /azure-iptables-monitor + - -v + - "3" + - -events=true + - -checkMap=true + - -ipv6=true + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: $AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY/azure-iptables-monitor:$AZURE_IPTABLES_MONITOR_TAG + imagePullPolicy: IfNotPresent + name: azure-iptables-monitor + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/config + name: iptables-config + - mountPath: /etc/config6 + name: ip6tables-config + - mountPath: /azure-block-iptables-bpf-map + name: iptables-block-bpf-map + readOnly: true + dnsPolicy: ClusterFirst + hostNetwork: true + initContainers: + - command: + - /azure-block-iptables + - -mode=attach + - -overwrite=true + image: $AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY/azure-iptables-monitor:$AZURE_IPTABLES_MONITOR_TAG + imagePullPolicy: IfNotPresent + name: iptables-blocker-init + resources: {} + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /sys/fs/bpf + name: bpf-maps + - mountPath: /proc + name: hostproc + - command: + - /azure-iptables-monitor + - -v + - "3" + - -events=true + - -terminateOnSuccess=true + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: $AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY/azure-iptables-monitor:$AZURE_IPTABLES_MONITOR_TAG + imagePullPolicy: IfNotPresent + name: azure-iptables-monitor-init + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/config + name: iptables-config + - command: + - sh + args: + - "/install-plugin.sh" + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium-distroless-init:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + name: install-cni-binaries + resources: {} + securityContext: + capabilities: + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-path + - command: + - sh + - -ec + - | + cp /usr/bin/cilium-mount /hostbin/cilium-mount; + nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; + rm /hostbin/cilium-mount + env: + - name: CGROUP_ROOT + value: /run/cilium/cgroupv2 + - name: BIN_PATH + value: /opt/cni/bin + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium-distroless-init:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + name: mount-cgroup + resources: {} + securityContext: + appArmorProfile: + type: Unconfined + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /hostproc + name: hostproc + - mountPath: /hostbin + name: cni-path + - command: + - sh + - -ec + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + env: + - name: BIN_PATH + value: /opt/cni/bin + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium-distroless-init:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + name: apply-sysctl-overwrites + resources: {} + securityContext: + appArmorProfile: + type: Unconfined + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /hostproc + name: hostproc + - mountPath: /hostbin + name: cni-path + - args: + - mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf + command: + - sh + - -c + - -- + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium-distroless-init:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + name: mount-bpf-fs + resources: {} + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + name: bpf-maps + - command: + - sh + args: + - "/init-container.sh" + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + key: clean-cilium-state + name: cilium-config + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + key: clean-cilium-bpf-state + name: cilium-config + optional: true + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium-distroless-init:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + name: clean-cilium-state + resources: + requests: + cpu: 100m + memory: 100Mi + securityContext: + appArmorProfile: + type: Unconfined + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /sys/fs/bpf + name: bpf-maps + - mountPath: /run/cilium/cgroupv2 + mountPropagation: HostToContainer + name: cilium-cgroup + - mountPath: /var/run/cilium + name: cilium-run + - command: + - /ipv6-hp-bpf + image: $IPV6_IMAGE_REGISTRY/ipv6-hp-bpf:$IPV6_HP_BPF_VERSION + imagePullPolicy: IfNotPresent + name: start-ipv6-hp-bpf + resources: {} + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/log + name: ipv6-hp-bpf + priorityClassName: system-node-critical + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccount: cilium + serviceAccountName: cilium + terminationGracePeriodSeconds: 1 + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + volumes: + - hostPath: + path: /var/log + type: DirectoryOrCreate + name: ipv6-hp-bpf + - hostPath: + path: /etc/systemd + type: DirectoryOrCreate + name: host-etc-systemd + - hostPath: + path: /lib/systemd + type: DirectoryOrCreate + name: host-lib-systemd + - hostPath: + path: /usr/lib + type: DirectoryOrCreate + name: host-usr-lib + - hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + name: cilium-run + - hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + name: bpf-maps + - hostPath: + path: /proc + type: Directory + name: hostproc + - hostPath: + path: /run/cilium/cgroupv2 + type: DirectoryOrCreate + name: cilium-cgroup + - hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + name: cni-path + - hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + name: etc-cni-netd + - hostPath: + path: /lib/modules + type: "" + name: lib-modules + - hostPath: + path: /run/xtables.lock + type: FileOrCreate + name: xtables-lock + - name: clustermesh-secrets + secret: + defaultMode: 256 + optional: true + secretName: cilium-clustermesh + - configMap: + defaultMode: 420 + name: cilium-config + name: cilium-config-path + - hostPath: + path: /proc/sys/net + type: Directory + name: host-proc-sys-net + - hostPath: + path: /proc/sys/kernel + type: Directory + name: host-proc-sys-kernel + - hostPath: + path: /var/run/netns + type: DirectoryOrCreate + name: cilium-netns + - configMap: + defaultMode: 420 + name: acns-flowlog-config + optional: true + name: hubble-flowlog-config + - hostPath: + path: /var/log/acns/hubble + type: DirectoryOrCreate + name: networkflowlogs + - configMap: + defaultMode: 420 + name: allowed-iptables-patterns + optional: true + name: iptables-config + - configMap: + defaultMode: 420 + name: allowed-ip6tables-patterns + optional: true + name: ip6tables-config + - hostPath: + path: /sys/fs/bpf/azure-block-iptables + type: DirectoryOrCreate + name: iptables-block-bpf-map + - configMap: + defaultMode: 420 + name: azure-dns-imds-ip-masq-agent-config + optional: true + name: azure-ip-masq-dir + updateStrategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 5% + type: RollingUpdate diff --git a/test/integration/manifests/cilium/v1.18/ebpf/podsubnet-dualstack/static/allowed-ip6tables-patterns.yaml b/test/integration/manifests/cilium/v1.18/ebpf/podsubnet-dualstack/static/allowed-ip6tables-patterns.yaml new file mode 100644 index 0000000000..b463118e2a --- /dev/null +++ b/test/integration/manifests/cilium/v1.18/ebpf/podsubnet-dualstack/static/allowed-ip6tables-patterns.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: allowed-ip6tables-patterns + namespace: kube-system +data: + filter: | + -A INPUT -p udp --dport 68 -j ACCEPT + + global: | + ^-N .* + ^-P .* + ^-A (KUBE-FIREWALL) .* + + ^-A .* -j (KUBE-FIREWALL) + ^-A IP-MASQ-AGENT + ^-A .* -j IP-MASQ-AGENT + + ^.*--comment.*cilium: + ^.*--comment.*cilium-feeder: + ^.*--comment.*AKS managed: added by AgentBaker + ^-A CILIUM_\S+ + mangle: | + + nat: | + -A POSTROUTING -j SWIFT + -A SWIFT -s + -A POSTROUTING -j SWIFT-POSTROUTING + -A SWIFT-POSTROUTING -s + raw: | + ^.*--comment.*localdns: skip conntrack + security: | diff --git a/test/integration/manifests/cilium/v1.18/ebpf/podsubnet-dualstack/static/azure-dns-imds-ip-masq-agent-config.yaml b/test/integration/manifests/cilium/v1.18/ebpf/podsubnet-dualstack/static/azure-dns-imds-ip-masq-agent-config.yaml new file mode 100644 index 0000000000..cfbf2fa1f0 --- /dev/null +++ b/test/integration/manifests/cilium/v1.18/ebpf/podsubnet-dualstack/static/azure-dns-imds-ip-masq-agent-config.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +data: + ip-masq-agent: "nonMasqueradeCIDRs:\n- 0.0.0.0/1\n- 128.0.0.0/3\n- 160.0.0.0/5\n- + 168.0.0.0/11\n- 168.32.0.0/12\n- 168.48.0.0/13\n- 168.56.0.0/14\n- 168.60.0.0/15\n- + 168.62.0.0/16\n- 168.63.0.0/17\n- 168.63.128.0/24\n- 168.63.129.0/29\n- 168.63.129.8/30\n- + 168.63.129.12/30\n- 168.63.129.17/32\n- 168.63.129.18/31\n- 168.63.129.20/30\n- + 168.63.129.24/29\n- 168.63.129.32/27\n- 168.63.129.64/26\n- 168.63.129.128/25\n- + 168.63.130.0/23\n- 168.63.132.0/22\n- 168.63.136.0/21\n- 168.63.144.0/20\n- 168.63.160.0/19\n- + 168.63.192.0/18\n- 168.64.0.0/10\n- 168.128.0.0/9\n- 169.0.0.0/9\n- 169.128.0.0/10\n- + 169.192.0.0/11\n- 169.224.0.0/12\n- 169.240.0.0/13\n- 169.248.0.0/14\n- 169.252.0.0/15\n- + 169.254.0.0/17\n- 169.254.128.0/19\n- 169.254.160.0/21\n- 169.254.168.0/24\n- + 169.254.169.0/25\n- 169.254.169.128/26\n- 169.254.169.192/27\n- 169.254.169.224/28\n- + 169.254.169.240/29\n- 169.254.169.248/30\n- 169.254.169.252/31\n- 169.254.169.255/32\n- + 169.254.170.0/23\n- 169.254.172.0/22\n- 169.254.176.0/20\n- 169.254.192.0/18\n- + 169.255.0.0/16\n- 170.0.0.0/7\n- 172.0.0.0/6\n- 176.0.0.0/4\n- 192.0.0.0/3\n- + 224.0.0.0/3\n- \"::/0\"\n \nmasqLinkLocal: true\nmasqLinkLocalIPv6: true\n" +kind: ConfigMap +metadata: + annotations: + meta.helm.sh/release-name: cilium + meta.helm.sh/release-namespace: kube-system + labels: + app.kubernetes.io/actually-managed-by: Eno + app.kubernetes.io/managed-by: Helm + name: azure-dns-imds-ip-masq-agent-config + namespace: kube-system diff --git a/test/integration/manifests/cilium/v1.18/ebpf/podsubnet-dualstack/static/cilium-config.yaml b/test/integration/manifests/cilium/v1.18/ebpf/podsubnet-dualstack/static/cilium-config.yaml new file mode 100644 index 0000000000..5d55e608fa --- /dev/null +++ b/test/integration/manifests/cilium/v1.18/ebpf/podsubnet-dualstack/static/cilium-config.yaml @@ -0,0 +1,174 @@ +apiVersion: v1 +data: + agent-not-ready-taint-key: node.cilium.io/agent-not-ready + arping-refresh-period: 30s + auto-direct-node-routes: "false" + bpf-algorithm-annotation: "false" + bpf-events-drop-enabled: "true" + bpf-events-policy-verdict-enabled: "true" + bpf-events-trace-enabled: "true" + bpf-lb-acceleration: disabled + bpf-lb-external-clusterip: "false" + bpf-lb-map-max: "65536" + bpf-lb-mode: snat + bpf-lb-mode-annotation: "false" + bpf-lb-sock: "false" + bpf-lb-sock-hostns-only: "true" + bpf-lb-sock-terminate-pod-connections: "false" + bpf-lb-source-range-all-types: "false" + bpf-map-dynamic-size-ratio: "0.0025" + bpf-policy-map-max: "16384" + bpf-root: /sys/fs/bpf + ces-slice-mode: fcfs + cgroup-root: /run/cilium/cgroupv2 + cilium-endpoint-gc-interval: 5m0s + cluster-id: "0" + cluster-name: alewpodsubebpfcilcanary + cni-exclusive: "false" + cni-log-file: /var/run/cilium/cilium-cni.log + datapath-mode: veth + debug: "false" + direct-routing-skip-unreachable: "false" + disable-cnp-status-updates: "true" + disable-embedded-dns-proxy: "false" + disable-endpoint-crd: "false" + dnsproxy-enable-transparent-mode: "false" + egress-gateway-reconciliation-trigger-interval: 1s + enable-auto-protect-node-port-range: "true" + enable-bgp-control-plane: "false" + enable-bpf-clock-probe: "true" + enable-bpf-masquerade: "true" + enable-cilium-endpoint-slice: "true" + enable-endpoint-health-checking: "false" + enable-endpoint-lockdown-on-policy-overflow: "false" + enable-endpoint-routes: "true" + enable-experimental-lb: "false" + enable-health-check-loadbalancer-ip: "false" + enable-health-check-nodeport: "true" + enable-health-checking: "true" + enable-host-legacy-routing: "false" + enable-hubble: "false" + enable-hubble-open-metrics: "false" + enable-internal-traffic-policy: "true" + enable-ip-masq-agent: "true" + enable-ipv4: "true" + enable-ipv4-big-tcp: "false" + enable-ipv4-masquerade: "true" + enable-ipv6: "true" + enable-ipv6-big-tcp: "false" + enable-ipv6-masquerade: "true" + enable-k8s-networkpolicy: "true" + enable-k8s-terminating-endpoint: "true" + enable-l2-neigh-discovery: "true" + enable-l7-proxy: "true" + enable-lb-ipam: "false" + enable-local-node-route: "false" + enable-local-redirect-policy: "true" + enable-masquerade-to-route-source: "false" + enable-metrics: "true" + enable-node-selector-labels: "false" + enable-non-default-deny-policies: "true" + enable-policy: default + enable-remote-node-identity: "true" + enable-remote-node-masquerade: "false" + enable-runtime-device-detection: "false" + enable-sctp: "false" + enable-session-affinity: "true" + enable-source-ip-verification: "false" + enable-standalone-dns-proxy: "true" + enable-svc-source-range-check: "true" + enable-tcx: "false" + enable-vtep: "false" + enable-well-known-identities: "false" + enable-wireguard: "false" + enable-xt-socket-fallback: "true" + external-envoy-proxy: "false" + health-check-icmp-failure-threshold: "3" + hubble-disable-tls: "false" + hubble-event-buffer-capacity: "4095" + hubble-export-file-max-backups: "5" + hubble-export-file-max-size-mb: "10" + hubble-flowlogs-config-path: /flowlog-config/flowlogs.yaml + hubble-listen-address: :4244 + hubble-metrics: flow:sourceEgressContext=pod;destinationIngressContext=pod tcp:sourceEgressContext=pod;destinationIngressContext=pod + drop:sourceEgressContext=pod;destinationIngressContext=pod dns:sourceEgressContext=pod;destinationIngressContext=pod + hubble-metrics-server: :9965 + hubble-socket-path: /var/run/cilium/hubble.sock + hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt + hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt + hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key + identity-allocation-mode: crd + install-iptables-rules: "true" + install-no-conntrack-iptables-rules: "false" + ipam: delegated-plugin + ipam-cilium-node-update-rate: 15s + ipv4-native-routing-cidr: 10.241.0.0/16 + ipv6-native-routing-cidr: fd00:aec6:6946:1::/64 + k8s-client-burst: "20" + k8s-client-qps: "10" + k8s-require-ipv4-pod-cidr: "false" + k8s-require-ipv6-pod-cidr: "false" + kube-proxy-replacement: "true" + kube-proxy-replacement-healthz-bind-address: 0.0.0.0:10256 + local-router-ipv4: 169.254.23.0 + mesh-auth-enabled: "false" + mesh-auth-gc-interval: 5m0s + mesh-auth-queue-size: "1024" + mesh-auth-rotated-identities-queue-size: "1024" + metrics: +cilium_bpf_map_pressure +cilium_proxy_datapath_update_timeout_total + monitor-aggregation: medium + monitor-aggregation-flags: all + monitor-aggregation-interval: 5s + nat-map-stats-entries: "32" + nat-map-stats-interval: 30s + node-port-bind-protection: "true" + nodeport-addresses: "" + nodes-gc-interval: 5m0s + operator-api-serve-addr: 127.0.0.1:9234 + operator-prometheus-serve-addr: :9963 + preallocate-bpf-maps: "false" + procfs: /host/proc + prometheus-serve-addr: :9962 + proxy-connect-timeout: "2" + proxy-idle-timeout-seconds: "60" + proxy-max-connection-duration-seconds: "0" + proxy-max-requests-per-connection: "0" + proxy-xff-num-trusted-hops-egress: "0" + proxy-xff-num-trusted-hops-ingress: "0" + remove-cilium-node-taints: "true" + routing-mode: native + set-cilium-is-up-condition: "true" + set-cilium-node-taints: "true" + sidecar-istio-proxy-image: cilium/istio_proxy + synchronize-k8s-nodes: "true" + tofqdns-dns-reject-response-code: refused + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "1000" + tofqdns-idle-connection-grace-period: 0s + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-min-ttl: "3600" + tofqdns-proxy-port: "40046" + tofqdns-proxy-response-max-delay: 100ms + tofqdns-server-port: "40045" + unmanaged-pod-watcher-interval: "0" + vtep-cidr: "" + vtep-endpoint: "" + vtep-mac: "" + vtep-mask: "" + ## new values for 1.18 + # bpf-policy-stats-map-max specifies the maximum number of entries in global + # policy stats map + bpf-policy-stats-map-max: "65536" + identity-management-mode: "agent" + tofqdns-preallocate-identities: "true" + policy-default-local-cluster: "false" +kind: ConfigMap +metadata: + annotations: + meta.helm.sh/release-name: cilium + meta.helm.sh/release-namespace: kube-system + labels: + app.kubernetes.io/actually-managed-by: Eno + app.kubernetes.io/managed-by: Helm + name: cilium-config + namespace: kube-system diff --git a/test/integration/manifests/cilium/v1.19/ebpf/podsubnet-dualstack/cilium.yaml b/test/integration/manifests/cilium/v1.19/ebpf/podsubnet-dualstack/cilium.yaml new file mode 100644 index 0000000000..a95f50120a --- /dev/null +++ b/test/integration/manifests/cilium/v1.19/ebpf/podsubnet-dualstack/cilium.yaml @@ -0,0 +1,458 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + annotations: + meta.helm.sh/release-name: cilium + meta.helm.sh/release-namespace: kube-system + labels: + app.kubernetes.io/actually-managed-by: Eno + app.kubernetes.io/managed-by: Helm + k8s-app: cilium + kubernetes.azure.com/managedby: aks + name: cilium + namespace: kube-system +spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: cilium + template: + metadata: + annotations: + prometheus.io/port: "9962" + prometheus.io/scrape: "true" + labels: + app.kubernetes.io/managed-by: Eno + k8s-app: cilium + kubernetes.azure.com/ebpf-dataplane: cilium + kubernetes.azure.com/managedby: aks + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: kubernetes.io/os + operator: In + values: + - linux + automountServiceAccountToken: true + containers: + - args: + - --config-dir=/tmp/cilium/config-map + command: + - cilium-agent + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: KUBE_CLIENT_BACKOFF_BASE + value: "1" + - name: KUBE_CLIENT_BACKOFF_DURATION + value: "120" + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium-distroless:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 10 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + - name: require-k8s-connectivity + value: "false" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + name: cilium-agent + ports: + - containerPort: 9962 + hostPort: 9962 + name: prometheus + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + resources: {} + securityContext: + appArmorProfile: + type: Unconfined + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + startupProbe: + failureThreshold: 105 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + - mountPath: /sys/fs/bpf + mountPropagation: HostToContainer + name: bpf-maps + - mountPath: /var/run/cilium + name: cilium-run + - mountPath: /host/etc/cni/net.d + name: etc-cni-netd + - mountPath: /var/lib/cilium/clustermesh + name: clustermesh-secrets + readOnly: true + - mountPath: /tmp/cilium/config-map + name: cilium-config-path + readOnly: true + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + - mountPath: /var/run/cilium/netns + mountPropagation: HostToContainer + name: cilium-netns + - mountPath: /flowlog-config + name: hubble-flowlog-config + readOnly: true + - mountPath: /var/log/acns/hubble + name: networkflowlogs + - mountPath: /etc/config + name: azure-ip-masq-dir + readOnly: true + - command: + - /azure-iptables-monitor + - -v + - "3" + - -events=true + - -checkMap=true + - -ipv6=true + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: $AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY/azure-iptables-monitor:$AZURE_IPTABLES_MONITOR_TAG + imagePullPolicy: IfNotPresent + name: azure-iptables-monitor + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/config + name: iptables-config + - mountPath: /etc/config6 + name: ip6tables-config + - mountPath: /azure-block-iptables-bpf-map + name: iptables-block-bpf-map + readOnly: true + dnsPolicy: ClusterFirst + hostNetwork: true + initContainers: + - command: + - /azure-block-iptables + - -mode=attach + - -overwrite=true + image: $AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY/azure-iptables-monitor:$AZURE_IPTABLES_MONITOR_TAG + imagePullPolicy: IfNotPresent + name: iptables-blocker-init + resources: {} + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /sys/fs/bpf + name: bpf-maps + - mountPath: /proc + name: hostproc + - command: + - /azure-iptables-monitor + - -v + - "3" + - -events=true + - -terminateOnSuccess=true + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: $AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY/azure-iptables-monitor:$AZURE_IPTABLES_MONITOR_TAG + imagePullPolicy: IfNotPresent + name: azure-iptables-monitor-init + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/config + name: iptables-config + # Merged init container: shares a single image pull and container start + # across the steps that previously ran in install-cni-binaries, + # mount-cgroup, apply-sysctl-overwrites, mount-bpf-fs and + # clean-cilium-state. Runs privileged because mount-bpf-fs needs + # Bidirectional mount propagation. + - name: cilium-init-all + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium-distroless-init:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -ec + - | + # 1. Install Cilium CNI plugin binaries on the host (was install-cni-binaries). + sh /install-plugin.sh + + # 2. Mount host cgroup2 via nsenter (was mount-cgroup). + cp /usr/bin/cilium-mount /hostbin/cilium-mount + nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" "$CGROUP_ROOT" + rm /hostbin/cilium-mount + + # 3. Apply sysctl overwrites on the host (was apply-sysctl-overwrites). + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix" + rm /hostbin/cilium-sysctlfix + + # 4. Mount bpf fs if not already mounted (was mount-bpf-fs). + mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf + + # 5. Clean cilium state if requested (was clean-cilium-state). + sh /init-container.sh + env: + - name: CGROUP_ROOT + value: /run/cilium/cgroupv2 + - name: BIN_PATH + value: /opt/cni/bin + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + key: clean-cilium-state + name: cilium-config + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + key: clean-cilium-bpf-state + name: cilium-config + optional: true + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + privileged: true + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-path + - mountPath: /hostbin + name: cni-path + - mountPath: /hostproc + name: hostproc + - mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + name: bpf-maps + - mountPath: /run/cilium/cgroupv2 + mountPropagation: HostToContainer + name: cilium-cgroup + - mountPath: /var/run/cilium + name: cilium-run + - mountPath: /host/etc/systemd + name: host-etc-systemd + - mountPath: /host/lib/systemd + name: host-lib-systemd + readOnly: true + - mountPath: /host/usr/lib + name: host-usr-lib + readOnly: true + resources: + requests: + cpu: 100m + memory: 100Mi + - command: + - /ipv6-hp-bpf + image: $IPV6_IMAGE_REGISTRY/ipv6-hp-bpf:$IPV6_HP_BPF_VERSION + imagePullPolicy: IfNotPresent + name: start-ipv6-hp-bpf + resources: {} + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/log + name: ipv6-hp-bpf + priorityClassName: system-node-critical + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccount: cilium + serviceAccountName: cilium + terminationGracePeriodSeconds: 1 + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + volumes: + - hostPath: + path: /var/log + type: DirectoryOrCreate + name: ipv6-hp-bpf + - hostPath: + path: /etc/systemd + type: DirectoryOrCreate + name: host-etc-systemd + - hostPath: + path: /lib/systemd + type: DirectoryOrCreate + name: host-lib-systemd + - hostPath: + path: /usr/lib + type: DirectoryOrCreate + name: host-usr-lib + - hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + name: cilium-run + - hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + name: bpf-maps + - hostPath: + path: /proc + type: Directory + name: hostproc + - hostPath: + path: /run/cilium/cgroupv2 + type: DirectoryOrCreate + name: cilium-cgroup + - hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + name: cni-path + - hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + name: etc-cni-netd + - hostPath: + path: /lib/modules + type: "" + name: lib-modules + - hostPath: + path: /run/xtables.lock + type: FileOrCreate + name: xtables-lock + - name: clustermesh-secrets + secret: + defaultMode: 256 + optional: true + secretName: cilium-clustermesh + - configMap: + defaultMode: 420 + name: cilium-config + name: cilium-config-path + - hostPath: + path: /proc/sys/net + type: Directory + name: host-proc-sys-net + - hostPath: + path: /proc/sys/kernel + type: Directory + name: host-proc-sys-kernel + - hostPath: + path: /var/run/netns + type: DirectoryOrCreate + name: cilium-netns + - configMap: + defaultMode: 420 + name: acns-flowlog-config + optional: true + name: hubble-flowlog-config + - hostPath: + path: /var/log/acns/hubble + type: DirectoryOrCreate + name: networkflowlogs + - configMap: + defaultMode: 420 + name: allowed-iptables-patterns + optional: true + name: iptables-config + - configMap: + defaultMode: 420 + name: allowed-ip6tables-patterns + optional: true + name: ip6tables-config + - hostPath: + path: /sys/fs/bpf/azure-block-iptables + type: DirectoryOrCreate + name: iptables-block-bpf-map + - configMap: + defaultMode: 420 + name: azure-dns-imds-ip-masq-agent-config + optional: true + name: azure-ip-masq-dir + updateStrategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 5% + type: RollingUpdate diff --git a/test/integration/manifests/cilium/v1.19/ebpf/podsubnet-dualstack/static/allowed-ip6tables-patterns.yaml b/test/integration/manifests/cilium/v1.19/ebpf/podsubnet-dualstack/static/allowed-ip6tables-patterns.yaml new file mode 100644 index 0000000000..b463118e2a --- /dev/null +++ b/test/integration/manifests/cilium/v1.19/ebpf/podsubnet-dualstack/static/allowed-ip6tables-patterns.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: allowed-ip6tables-patterns + namespace: kube-system +data: + filter: | + -A INPUT -p udp --dport 68 -j ACCEPT + + global: | + ^-N .* + ^-P .* + ^-A (KUBE-FIREWALL) .* + + ^-A .* -j (KUBE-FIREWALL) + ^-A IP-MASQ-AGENT + ^-A .* -j IP-MASQ-AGENT + + ^.*--comment.*cilium: + ^.*--comment.*cilium-feeder: + ^.*--comment.*AKS managed: added by AgentBaker + ^-A CILIUM_\S+ + mangle: | + + nat: | + -A POSTROUTING -j SWIFT + -A SWIFT -s + -A POSTROUTING -j SWIFT-POSTROUTING + -A SWIFT-POSTROUTING -s + raw: | + ^.*--comment.*localdns: skip conntrack + security: | diff --git a/test/integration/manifests/cilium/v1.19/ebpf/podsubnet-dualstack/static/azure-dns-imds-ip-masq-agent-config.yaml b/test/integration/manifests/cilium/v1.19/ebpf/podsubnet-dualstack/static/azure-dns-imds-ip-masq-agent-config.yaml new file mode 100644 index 0000000000..cfbf2fa1f0 --- /dev/null +++ b/test/integration/manifests/cilium/v1.19/ebpf/podsubnet-dualstack/static/azure-dns-imds-ip-masq-agent-config.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +data: + ip-masq-agent: "nonMasqueradeCIDRs:\n- 0.0.0.0/1\n- 128.0.0.0/3\n- 160.0.0.0/5\n- + 168.0.0.0/11\n- 168.32.0.0/12\n- 168.48.0.0/13\n- 168.56.0.0/14\n- 168.60.0.0/15\n- + 168.62.0.0/16\n- 168.63.0.0/17\n- 168.63.128.0/24\n- 168.63.129.0/29\n- 168.63.129.8/30\n- + 168.63.129.12/30\n- 168.63.129.17/32\n- 168.63.129.18/31\n- 168.63.129.20/30\n- + 168.63.129.24/29\n- 168.63.129.32/27\n- 168.63.129.64/26\n- 168.63.129.128/25\n- + 168.63.130.0/23\n- 168.63.132.0/22\n- 168.63.136.0/21\n- 168.63.144.0/20\n- 168.63.160.0/19\n- + 168.63.192.0/18\n- 168.64.0.0/10\n- 168.128.0.0/9\n- 169.0.0.0/9\n- 169.128.0.0/10\n- + 169.192.0.0/11\n- 169.224.0.0/12\n- 169.240.0.0/13\n- 169.248.0.0/14\n- 169.252.0.0/15\n- + 169.254.0.0/17\n- 169.254.128.0/19\n- 169.254.160.0/21\n- 169.254.168.0/24\n- + 169.254.169.0/25\n- 169.254.169.128/26\n- 169.254.169.192/27\n- 169.254.169.224/28\n- + 169.254.169.240/29\n- 169.254.169.248/30\n- 169.254.169.252/31\n- 169.254.169.255/32\n- + 169.254.170.0/23\n- 169.254.172.0/22\n- 169.254.176.0/20\n- 169.254.192.0/18\n- + 169.255.0.0/16\n- 170.0.0.0/7\n- 172.0.0.0/6\n- 176.0.0.0/4\n- 192.0.0.0/3\n- + 224.0.0.0/3\n- \"::/0\"\n \nmasqLinkLocal: true\nmasqLinkLocalIPv6: true\n" +kind: ConfigMap +metadata: + annotations: + meta.helm.sh/release-name: cilium + meta.helm.sh/release-namespace: kube-system + labels: + app.kubernetes.io/actually-managed-by: Eno + app.kubernetes.io/managed-by: Helm + name: azure-dns-imds-ip-masq-agent-config + namespace: kube-system diff --git a/test/integration/manifests/cilium/v1.19/ebpf/podsubnet-dualstack/static/cilium-config.yaml b/test/integration/manifests/cilium/v1.19/ebpf/podsubnet-dualstack/static/cilium-config.yaml new file mode 100644 index 0000000000..0664af2345 --- /dev/null +++ b/test/integration/manifests/cilium/v1.19/ebpf/podsubnet-dualstack/static/cilium-config.yaml @@ -0,0 +1,174 @@ +apiVersion: v1 +data: + agent-not-ready-taint-key: node.cilium.io/agent-not-ready + arping-refresh-period: 30s + auto-direct-node-routes: "false" + bpf-algorithm-annotation: "false" + bpf-events-drop-enabled: "true" + bpf-events-policy-verdict-enabled: "true" + bpf-events-trace-enabled: "true" + bpf-lb-acceleration: disabled + bpf-lb-external-clusterip: "false" + bpf-lb-map-max: "65536" + bpf-lb-mode: snat + bpf-lb-mode-annotation: "false" + bpf-lb-sock: "false" + bpf-lb-sock-hostns-only: "true" + bpf-lb-sock-terminate-pod-connections: "false" + bpf-lb-source-range-all-types: "false" + bpf-map-dynamic-size-ratio: "0.0025" + bpf-policy-map-max: "16384" + bpf-root: /sys/fs/bpf + ces-slice-mode: fcfs + cgroup-root: /run/cilium/cgroupv2 + cilium-endpoint-gc-interval: 5m0s + cluster-id: "0" + cluster-name: alewpodsubebpfcilcanary + cni-exclusive: "false" + cni-log-file: /var/run/cilium/cilium-cni.log + datapath-mode: veth + debug: "false" + direct-routing-skip-unreachable: "false" + disable-cnp-status-updates: "true" + disable-embedded-dns-proxy: "false" + disable-endpoint-crd: "false" + dnsproxy-enable-transparent-mode: "false" + egress-gateway-reconciliation-trigger-interval: 1s + enable-auto-protect-node-port-range: "true" + enable-bgp-control-plane: "false" + enable-bpf-clock-probe: "true" + enable-bpf-masquerade: "true" + enable-cilium-endpoint-slice: "true" + enable-endpoint-health-checking: "false" + enable-endpoint-lockdown-on-policy-overflow: "false" + enable-endpoint-routes: "true" + enable-experimental-lb: "false" + enable-health-check-loadbalancer-ip: "false" + enable-health-check-nodeport: "true" + enable-health-checking: "true" + enable-host-legacy-routing: "false" + enable-hubble: "false" + enable-hubble-open-metrics: "false" + enable-internal-traffic-policy: "true" + enable-ip-masq-agent: "true" + enable-ipv4: "true" + enable-ipv4-big-tcp: "false" + enable-ipv4-masquerade: "true" + enable-ipv6: "true" + enable-ipv6-big-tcp: "false" + enable-ipv6-masquerade: "true" + enable-k8s-networkpolicy: "true" + enable-k8s-terminating-endpoint: "true" + enable-l2-neigh-discovery: "true" + enable-l7-proxy: "true" + enable-lb-ipam: "false" + enable-local-node-route: "false" + enable-local-redirect-policy: "true" + enable-masquerade-to-route-source: "false" + enable-metrics: "true" + enable-node-selector-labels: "false" + enable-non-default-deny-policies: "true" + enable-policy: default + enable-remote-node-identity: "true" + enable-remote-node-masquerade: "false" + enable-runtime-device-detection: "false" + enable-sctp: "false" + enable-session-affinity: "true" + enable-source-ip-verification: "false" + enable-standalone-dns-proxy: "true" + enable-svc-source-range-check: "true" + enable-tcx: "false" + enable-vtep: "false" + enable-well-known-identities: "false" + enable-wireguard: "false" + enable-xt-socket-fallback: "true" + external-envoy-proxy: "false" + health-check-icmp-failure-threshold: "3" + hubble-disable-tls: "false" + hubble-event-buffer-capacity: "4095" + hubble-export-file-max-backups: "5" + hubble-export-file-max-size-mb: "10" + hubble-flowlogs-config-path: /flowlog-config/flowlogs.yaml + hubble-listen-address: :4244 + hubble-metrics: flow:sourceEgressContext=pod;destinationIngressContext=pod tcp:sourceEgressContext=pod;destinationIngressContext=pod + drop:sourceEgressContext=pod;destinationIngressContext=pod dns:sourceEgressContext=pod;destinationIngressContext=pod + hubble-metrics-server: :9965 + hubble-socket-path: /var/run/cilium/hubble.sock + hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt + hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt + hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key + identity-allocation-mode: crd + install-iptables-rules: "true" + install-no-conntrack-iptables-rules: "false" + ipam: delegated-plugin + ipam-cilium-node-update-rate: 15s + ipv4-native-routing-cidr: 10.241.0.0/16 + ipv6-native-routing-cidr: fd00:aec6:6946:1::/64 + k8s-client-burst: "20" + k8s-client-qps: "10" + k8s-require-ipv4-pod-cidr: "false" + k8s-require-ipv6-pod-cidr: "false" + kube-proxy-replacement: "true" + kube-proxy-replacement-healthz-bind-address: 0.0.0.0:10256 + local-router-ipv4: 169.254.23.0 + mesh-auth-enabled: "false" + mesh-auth-gc-interval: 5m0s + mesh-auth-queue-size: "1024" + mesh-auth-rotated-identities-queue-size: "1024" + metrics: +cilium_bpf_map_pressure +cilium_proxy_datapath_update_timeout_total + monitor-aggregation: medium + monitor-aggregation-flags: all + monitor-aggregation-interval: 5s + nat-map-stats-entries: "32" + nat-map-stats-interval: 30s + node-port-bind-protection: "true" + nodeport-addresses: "" + nodes-gc-interval: 5m0s + operator-api-serve-addr: 127.0.0.1:9234 + operator-prometheus-serve-addr: :9963 + preallocate-bpf-maps: "false" + procfs: /host/proc + prometheus-serve-addr: :9962 + proxy-connect-timeout: "2" + proxy-idle-timeout-seconds: "60" + proxy-max-connection-duration-seconds: "0" + proxy-max-requests-per-connection: "0" + proxy-xff-num-trusted-hops-egress: "0" + proxy-xff-num-trusted-hops-ingress: "0" + remove-cilium-node-taints: "true" + routing-mode: native + set-cilium-is-up-condition: "true" + set-cilium-node-taints: "true" + sidecar-istio-proxy-image: cilium/istio_proxy + synchronize-k8s-nodes: "true" + tofqdns-dns-reject-response-code: refused + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "1000" + tofqdns-idle-connection-grace-period: 0s + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-min-ttl: "3600" + tofqdns-proxy-port: "40046" + tofqdns-proxy-response-max-delay: 100ms + tofqdns-server-port: "40045" + unmanaged-pod-watcher-interval: "0s" + vtep-cidr: "" + vtep-endpoint: "" + vtep-mac: "" + vtep-mask: "" + ## new values for 1.18 + # bpf-policy-stats-map-max specifies the maximum number of entries in global + # policy stats map + bpf-policy-stats-map-max: "65536" + identity-management-mode: "operator" + tofqdns-preallocate-identities: "true" + policy-default-local-cluster: "false" +kind: ConfigMap +metadata: + annotations: + meta.helm.sh/release-name: cilium + meta.helm.sh/release-namespace: kube-system + labels: + app.kubernetes.io/actually-managed-by: Eno + app.kubernetes.io/managed-by: Helm + name: cilium-config + namespace: kube-system