From a48789d514e7f63143d18f5aa55a509074d2d047 Mon Sep 17 00:00:00 2001 From: Alain Kaeslin Date: Thu, 30 Oct 2025 10:57:13 +0100 Subject: [PATCH 1/7] Fix test. --- pkg/cloudscale_ccm/reconcile_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cloudscale_ccm/reconcile_test.go b/pkg/cloudscale_ccm/reconcile_test.go index bbc61e5..661154e 100644 --- a/pkg/cloudscale_ccm/reconcile_test.go +++ b/pkg/cloudscale_ccm/reconcile_test.go @@ -161,8 +161,8 @@ func TestDesiredService(t *testing.T) { assert.Equal(t, desired.pools[0].Protocol, "tcp") assert.Equal(t, desired.pools[0].Algorithm, "round_robin") assert.Equal(t, desired.pools[1].Name, "tcp/https") - assert.Equal(t, desired.pools[0].Protocol, "tcp") - assert.Equal(t, desired.pools[0].Algorithm, "round_robin") + assert.Equal(t, desired.pools[1].Protocol, "tcp") + assert.Equal(t, desired.pools[1].Algorithm, "round_robin") // One member per server for _, pool := range desired.pools { From 8ff5ce31d32b0f47dcff6c376c563f87830e7b1e Mon Sep 17 00:00:00 2001 From: Alain Kaeslin Date: Thu, 30 Oct 2025 14:09:33 +0100 Subject: [PATCH 2/7] Use more "common" values for NodePort in test for improved test readability. --- pkg/cloudscale_ccm/reconcile_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/cloudscale_ccm/reconcile_test.go b/pkg/cloudscale_ccm/reconcile_test.go index 661154e..8c65042 100644 --- a/pkg/cloudscale_ccm/reconcile_test.go +++ b/pkg/cloudscale_ccm/reconcile_test.go @@ -137,13 +137,13 @@ func TestDesiredService(t *testing.T) { { Protocol: "TCP", Port: 80, - NodePort: 8080, + NodePort: 30080, Name: "http", }, { Protocol: "TCP", Port: 443, - NodePort: 8443, + NodePort: 30443, Name: "https", }, } @@ -173,11 +173,11 @@ func TestDesiredService(t *testing.T) { assert.Equal(t, "10.0.0.2", members[1].Address) assert.True(t, - members[0].ProtocolPort == 8443 || - members[0].ProtocolPort == 8080) + members[0].ProtocolPort == 30443 || + members[0].ProtocolPort == 30080) assert.True(t, - members[1].ProtocolPort == 8443 || - members[1].ProtocolPort == 8080) + members[1].ProtocolPort == 30443 || + members[1].ProtocolPort == 30080) } // One listener per pool @@ -864,7 +864,7 @@ func TestLimitSubnets(t *testing.T) { { Protocol: "TCP", Port: 80, - NodePort: 123456, + NodePort: 30080, }, } From 5a76f1cffd372ea19d44d4b451340538badd040c Mon Sep 17 00:00:00 2001 From: Alain Kaeslin Date: Thu, 30 Oct 2025 12:57:30 +0100 Subject: [PATCH 3/7] Add UDP support. --- examples/dns-dual-protocol.yaml | 94 ++++++ examples/udp.yaml | 63 ++++ pkg/cloudscale_ccm/loadbalancer.go | 15 +- pkg/cloudscale_ccm/reconcile.go | 25 +- pkg/cloudscale_ccm/reconcile_test.go | 207 +++++++++++++ pkg/internal/integration/service_test.go | 360 +++++++++++++++++++++-- 6 files changed, 723 insertions(+), 41 deletions(-) create mode 100644 examples/dns-dual-protocol.yaml create mode 100644 examples/udp.yaml diff --git a/examples/dns-dual-protocol.yaml b/examples/dns-dual-protocol.yaml new file mode 100644 index 0000000..f69f476 --- /dev/null +++ b/examples/dns-dual-protocol.yaml @@ -0,0 +1,94 @@ +# Deploys a DNS server container and creates a loadbalancer service for it with both UDP and TCP: +# +# export KUBECONFIG=path/to/kubeconfig +# kubectl apply -f dns-dual-protocol.yaml +# +# Wait for `kubectl describe service dns-server` to show "Loadbalancer Ensured", +# then use the IP address found under "LoadBalancer Ingress" to connect to the +# service. +# +# You can test the DNS service with dig: +# +# # Test UDP (default) +# dig @$(kubectl get service dns-server -o jsonpath='{.status.loadBalancer.ingress[0].ip}') example.com +# +# # Test TCP explicitly +# dig +tcp @$(kubectl get service dns-server -o jsonpath='{.status.loadBalancer.ingress[0].ip}') example.com +# +# To view the logs: +# +# kubectl logs -l "app=dns-server" +# +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns-config +data: + Corefile: | + .:53 { + log + errors + health + ready + whoami + forward . 8.8.8.8 9.9.9.9 + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dns-server +spec: + replicas: 2 + selector: + matchLabels: + app: dns-server + template: + metadata: + labels: + app: dns-server + spec: + containers: + - name: coredns + image: coredns/coredns:1.11.1 + args: + - -conf + - /etc/coredns/Corefile + volumeMounts: + - name: config + mountPath: /etc/coredns + ports: + - containerPort: 53 + protocol: UDP + name: dns-udp + - containerPort: 53 + protocol: TCP + name: dns-tcp + volumes: + - name: config + configMap: + name: coredns-config +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + k8s.cloudscale.ch/loadbalancer-health-monitor-delay-s: "3" + k8s.cloudscale.ch/loadbalancer-health-monitor-timeout-s: "2" + labels: + app: dns-server + name: dns-server +spec: + ports: + - port: 53 + protocol: UDP + targetPort: 53 + name: dns-udp + - port: 53 + protocol: TCP + targetPort: 53 + name: dns-tcp + selector: + app: dns-server + type: LoadBalancer diff --git a/examples/udp.yaml b/examples/udp.yaml new file mode 100644 index 0000000..a115b71 --- /dev/null +++ b/examples/udp.yaml @@ -0,0 +1,63 @@ +# Deploys a UDP echo server container and creates a loadbalancer service for it: +# +# export KUBECONFIG=path/to/kubeconfig +# kubectl apply -f udp-echo.yml +# +# Wait for `kubectl describe service udp-echo` to show "Loadbalancer Ensured", +# then use the IP address found under "LoadBalancer Ingress" to connect to the +# service. +# +# You can test the UDP service with netcat: +# +# echo "Tell me a joke" | nc -u -w 1 $(kubectl get service udp-echo -o jsonpath='{.status.loadBalancer.ingress[0].ip}') 5000 +# +# To view the logs: +# +# kubectl logs -l "app=udp-echo" +# +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: udp-echo +spec: + replicas: 2 + selector: + matchLabels: + app: udp-echo + template: + metadata: + labels: + app: udp-echo + spec: + containers: + - name: udp-echo + image: docker.io/alpine/socat + command: + - socat + - "-v" + - "UDP4-RECVFROM:5353,fork" + - "SYSTEM:echo 'I could tell you a UDP joke, but you might not get it...',pipes" + ports: + - containerPort: 5353 + protocol: UDP +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + k8s.cloudscale.ch/loadbalancer-health-monitor-type: udp-connect + k8s.cloudscale.ch/loadbalancer-health-monitor-delay-s: "3" + k8s.cloudscale.ch/loadbalancer-health-monitor-timeout-s: "2" + labels: + app: udp-echo + name: udp-echo +spec: + ports: + - port: 5000 + protocol: UDP + targetPort: 5353 + name: udp + selector: + app: udp-echo + type: LoadBalancer diff --git a/pkg/cloudscale_ccm/loadbalancer.go b/pkg/cloudscale_ccm/loadbalancer.go index f535f44..4cb8dc1 100644 --- a/pkg/cloudscale_ccm/loadbalancer.go +++ b/pkg/cloudscale_ccm/loadbalancer.go @@ -115,8 +115,14 @@ const ( // as all pools have to be recreated. LoadBalancerPoolAlgorithm = "k8s.cloudscale.ch/loadbalancer-pool-algorithm" - // LoadBalancerPoolProtocol defines the protocol for all the pools of the - // service. We are technically able to have different protocols for + // LoadBalancerPoolProtocol defines the protocol the pools of a port + // with protocol `TCP` use. Set it to `proxy` and `proxyv2` if TCP + // traffic should be forwarded using these protocols. + // + // When setting the protocol of a port to `UDP,` traffic is always forwarded + // using UDP. + // + // We are technically able to have different protocols for // different ports in a service, but as our options apart from `tcp` are // currently `proxy` and `proxyv2`, we go with Kubernetes's recommendation // to apply these protocols to all incoming connections the same way: @@ -216,6 +222,8 @@ const ( // LoadBalancerHealthMonitorType defines the approach the monitor takes. // (ping, tcp, http, https, tls-hello). // + // Note that the same type is used for all ports in your service. + // // See https://www.cloudscale.ch/en/api/v1#health-monitor-types // // Changing this annotation on an active service may lead to new @@ -233,6 +241,9 @@ const ( // LoadBalancerListenerProtocol defines the protocol used by the listening // port on the loadbalancer. Currently, only tcp is supported. // + // This property is ignored for ports with the protocol `UDP`, where + // we always create udp listeners. + // // See https://www.cloudscale.ch/en/api/v1#listener-protocols // // Changing this annotation on an established service may cause downtime diff --git a/pkg/cloudscale_ccm/reconcile.go b/pkg/cloudscale_ccm/reconcile.go index 54ec025..a387c86 100644 --- a/pkg/cloudscale_ccm/reconcile.go +++ b/pkg/cloudscale_ccm/reconcile.go @@ -122,9 +122,10 @@ func desiredLbState( for _, port := range serviceInfo.Service.Spec.Ports { - if port.Protocol != "TCP" { + if port.Protocol != v1.ProtocolTCP && port.Protocol != v1.ProtocolUDP { return nil, fmt.Errorf( - "service %s: cannot use %s for %d, only TCP is supported", + "service %s: cannot use %s for %d"+ + ", only TCP and UDP are supported", serviceInfo.Service.Name, port.Protocol, port.Port) @@ -145,10 +146,15 @@ func desiredLbState( } } + poolProtocol := protocol + if port.Protocol != v1.ProtocolTCP { + poolProtocol = "udp" + } + pool := cloudscale.LoadBalancerPool{ Name: poolName(port.Protocol, port.Name), Algorithm: algorithm, - Protocol: protocol, + Protocol: poolProtocol, } s.pools = append(s.pools, &pool) @@ -213,7 +219,7 @@ func desiredLbState( s.monitors[&pool] = append(s.monitors[&pool], *monitor) // Add a listener for each pool - listener, err := listenerForPort(serviceInfo, int(port.Port)) + listener, err := listenerForPort(serviceInfo, port) if err != nil { return nil, err } @@ -813,7 +819,7 @@ func runActions( // annotations into consideration. func listenerForPort( serviceInfo *serviceInfo, - port int, + port v1.ServicePort, ) (*cloudscale.LoadBalancerListener, error) { var ( @@ -821,8 +827,13 @@ func listenerForPort( err error ) - listener.Protocol = serviceInfo.annotation(LoadBalancerListenerProtocol) - listener.ProtocolPort = port + listenerProtocol := serviceInfo.annotation(LoadBalancerListenerProtocol) + if port.Protocol != v1.ProtocolTCP { + listenerProtocol = "udp" + } + + listener.Protocol = listenerProtocol + listener.ProtocolPort = int(port.Port) listener.Name = listenerName(listener.Protocol, listener.ProtocolPort) listener.TimeoutClientDataMS, err = serviceInfo.annotationInt( diff --git a/pkg/cloudscale_ccm/reconcile_test.go b/pkg/cloudscale_ccm/reconcile_test.go index 8c65042..84c9e9a 100644 --- a/pkg/cloudscale_ccm/reconcile_test.go +++ b/pkg/cloudscale_ccm/reconcile_test.go @@ -199,6 +199,213 @@ func TestDesiredService(t *testing.T) { } } +func TestDesiredServiceUDP(t *testing.T) { + t.Parallel() + + s := testkit.NewService("service").V1() + i := newServiceInfo(s, "") + + nodes := []*v1.Node{ + testkit.NewNode("worker-1").V1(), + testkit.NewNode("worker-2").V1(), + } + + servers := []cloudscale.Server{ + { + Name: "worker-1", + ZonalResource: cloudscale.ZonalResource{ + Zone: cloudscale.Zone{Slug: "rma1"}, + }, + Interfaces: []cloudscale.Interface{{ + Addresses: []cloudscale.Address{{ + Address: "10.0.0.1", + Subnet: cloudscale.SubnetStub{ + UUID: "00000000-0000-0000-0000-000000000000", + }, + }}, + }}, + }, + { + Name: "worker-2", + ZonalResource: cloudscale.ZonalResource{ + Zone: cloudscale.Zone{Slug: "rma1"}, + }, + Interfaces: []cloudscale.Interface{{ + Addresses: []cloudscale.Address{{ + Address: "10.0.0.2", + Subnet: cloudscale.SubnetStub{ + UUID: "00000000-0000-0000-0000-000000000000", + }, + }}, + }}, + }, + } + + s.Annotations = map[string]string{ + LoadBalancerHealthMonitorType: "udp-connect", + LoadBalancerHealthMonitorDelayS: "3", + LoadBalancerHealthMonitorTimeoutS: "2", + } + s.Spec.Ports = []v1.ServicePort{ + { + Protocol: "UDP", + Port: 53, + NodePort: 30053, + Name: "udp", + }, + } + + desired, err := desiredLbState(i, nodes, servers) + assert.NoError(t, err) + + // Ensure the lb exists + assert.Equal(t, "lb-standard", desired.lb.Flavor.Slug) + assert.Len(t, desired.lb.VIPAddresses, 0) + + // Have one pool per service port + assert.Len(t, desired.pools, 1) + assert.Equal(t, desired.pools[0].Name, "udp/udp") + assert.Equal(t, desired.pools[0].Protocol, "udp") + assert.Equal(t, desired.pools[0].Algorithm, "round_robin") + + // One member per server + for _, pool := range desired.pools { + members := desired.members[pool] + assert.Len(t, members, 2) + + assert.Equal(t, "10.0.0.1", members[0].Address) + assert.Equal(t, "10.0.0.2", members[1].Address) + + assert.True(t, members[0].ProtocolPort == 30053) + } + + // One listener per pool + for _, pool := range desired.pools { + listeners := desired.listeners[pool] + assert.Len(t, listeners, 1) + + assert.Equal(t, "udp", listeners[0].Protocol) + assert.True(t, listeners[0].ProtocolPort == 53) + } + + // One health monitor per pool + for _, pool := range desired.pools { + monitors := desired.monitors[pool] + assert.Len(t, monitors, 1) + assert.Equal(t, "udp-connect", monitors[0].Type) + assert.Equal(t, 3, monitors[0].DelayS) + assert.Equal(t, 2, monitors[0].TimeoutS) + } +} + +func TestDesiredServiceDualProtocol(t *testing.T) { + t.Parallel() + + s := testkit.NewService("service").V1() + i := newServiceInfo(s, "") + + nodes := []*v1.Node{ + testkit.NewNode("worker-1").V1(), + testkit.NewNode("worker-2").V1(), + } + + servers := []cloudscale.Server{ + { + Name: "worker-1", + ZonalResource: cloudscale.ZonalResource{ + Zone: cloudscale.Zone{Slug: "rma1"}, + }, + Interfaces: []cloudscale.Interface{{ + Addresses: []cloudscale.Address{{ + Address: "10.0.0.1", + Subnet: cloudscale.SubnetStub{ + UUID: "00000000-0000-0000-0000-000000000000", + }, + }}, + }}, + }, + { + Name: "worker-2", + ZonalResource: cloudscale.ZonalResource{ + Zone: cloudscale.Zone{Slug: "rma1"}, + }, + Interfaces: []cloudscale.Interface{{ + Addresses: []cloudscale.Address{{ + Address: "10.0.0.2", + Subnet: cloudscale.SubnetStub{ + UUID: "00000000-0000-0000-0000-000000000000", + }, + }}, + }}, + }, + } + + s.Annotations = map[string]string{ + LoadBalancerHealthMonitorType: "udp-connect", + LoadBalancerHealthMonitorDelayS: "3", + LoadBalancerHealthMonitorTimeoutS: "2", + } + s.Spec.Ports = []v1.ServicePort{ + { + Protocol: "UDP", + Port: 53, + NodePort: 30053, + Name: "udp", + }, + { + Protocol: "TCP", + Port: 53, + NodePort: 30053, + Name: "tcp", + }, + } + + desired, err := desiredLbState(i, nodes, servers) + assert.NoError(t, err) + + // Ensure the lb exists + assert.Equal(t, "lb-standard", desired.lb.Flavor.Slug) + assert.Len(t, desired.lb.VIPAddresses, 0) + + // Have one pool per service port + assert.Len(t, desired.pools, 2) + assert.Equal(t, desired.pools[0].Name, "udp/udp") + assert.Equal(t, desired.pools[0].Protocol, "udp") + assert.Equal(t, desired.pools[0].Algorithm, "round_robin") + assert.Equal(t, desired.pools[1].Name, "tcp/tcp") + assert.Equal(t, desired.pools[1].Protocol, "tcp") + assert.Equal(t, desired.pools[1].Algorithm, "round_robin") + + // One member per server + for _, pool := range desired.pools { + members := desired.members[pool] + assert.Len(t, members, 2) + + assert.Equal(t, "10.0.0.1", members[0].Address) + assert.Equal(t, "10.0.0.2", members[1].Address) + + assert.True(t, members[0].ProtocolPort == 30053) + } + + // One listener per pool + for _, pool := range desired.pools { + listeners := desired.listeners[pool] + assert.Len(t, listeners, 1) + + assert.Equal(t, pool.Protocol, listeners[0].Protocol) + assert.True(t, listeners[0].ProtocolPort == 53) + } + + // One health monitor per pool + for _, pool := range desired.pools { + monitors := desired.monitors[pool] + assert.Len(t, monitors, 1) + assert.Equal(t, "udp-connect", monitors[0].Type) + assert.Equal(t, 3, monitors[0].DelayS) + assert.Equal(t, 2, monitors[0].TimeoutS) + } +} + func TestActualState(t *testing.T) { t.Parallel() diff --git a/pkg/internal/integration/service_test.go b/pkg/internal/integration/service_test.go index b5ffd61..d19befc 100644 --- a/pkg/internal/integration/service_test.go +++ b/pkg/internal/integration/service_test.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "math/rand" + "net" "net/netip" "os" "os/exec" @@ -27,7 +28,7 @@ import ( ) func (s *IntegrationTestSuite) CreateDeployment( - name string, image string, replicas int32, port int32, args ...string) { + name string, image string, replicas int32, protocol v1.Protocol, port int32, args ...string) { var command []string @@ -57,7 +58,7 @@ func (s *IntegrationTestSuite) CreateDeployment( Command: command, Args: args, Ports: []v1.ContainerPort{ - {ContainerPort: port}, + {ContainerPort: port, Protocol: protocol}, }, }, }, @@ -77,21 +78,47 @@ func (s *IntegrationTestSuite) CreateDeployment( s.Require().NoError(err) } +func (s *IntegrationTestSuite) CreateConfigMap(name string, data map[string]string) { + _, err := s.k8s.CoreV1().ConfigMaps(s.ns).Create( + context.Background(), + &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Data: data, + }, + metav1.CreateOptions{}, + ) + + s.Require().NoError(err) +} + +// ServicePortSpec defines the configuration for a single service port +type ServicePortSpec struct { + Protocol v1.Protocol + Port int32 + TargetPort int32 +} + func (s *IntegrationTestSuite) ExposeDeployment( - name string, port int32, targetPort int32, annotations map[string]string) { + name string, annotations map[string]string, ports ...ServicePortSpec) { + + servicePorts := make([]v1.ServicePort, len(ports)) + for i, p := range ports { + servicePorts[i] = v1.ServicePort{ + Name: fmt.Sprintf("port%d", i), + Protocol: p.Protocol, + Port: p.Port, + TargetPort: intstr.FromInt32(p.TargetPort), + } + } spec := v1.ServiceSpec{ Type: v1.ServiceTypeLoadBalancer, Selector: map[string]string{ "app": name, }, - Ports: []v1.ServicePort{ - { - Protocol: v1.ProtocolTCP, - Port: port, - TargetPort: intstr.FromInt32(targetPort), - }, - }, + Ports: servicePorts, } service, err := s.k8s.CoreV1().Services(s.ns).Get( @@ -281,10 +308,11 @@ func (s *IntegrationTestSuite) TestServiceEndToEnd() { // Deploy a TCP server that returns the hostname s.T().Log("Creating nginx deployment") - s.CreateDeployment("nginx", "nginxdemos/hello:plain-text", 2, 80) + s.CreateDeployment("nginx", "nginxdemos/hello:plain-text", 2, v1.ProtocolTCP, 80) // Expose the deployment using a LoadBalancer service - s.ExposeDeployment("nginx", 80, 80, nil) + s.ExposeDeployment("nginx", nil, + ServicePortSpec{Protocol: v1.ProtocolTCP, Port: 80, TargetPort: 80}) // Wait for the service to be ready s.T().Log("Waiting for nginx service to be ready") @@ -338,6 +366,274 @@ func (s *IntegrationTestSuite) TestServiceEndToEnd() { s.Assert().NotContains(lines, "Warn") } +func (s *IntegrationTestSuite) TestServiceEndToEndUDP() { + + // Note the start for the log + start := time.Now() + + // Deploy a UDP echo server + s.T().Log("Creating udp-echo deployment") + s.CreateDeployment("udp-echo", "docker.io/alpine/socat", 2, v1.ProtocolUDP, 5353, + "socat", + "-v", + "UDP4-RECVFROM:5353,fork", + "SYSTEM:echo 'I could tell you a UDP joke, but you might not get it...',pipes", + ) + + // Expose the deployment using a LoadBalancer service with UDP annotations + s.ExposeDeployment("udp-echo", map[string]string{ + "k8s.cloudscale.ch/loadbalancer-health-monitor-type": "udp-connect", + "k8s.cloudscale.ch/loadbalancer-health-monitor-delay-s": "3", + "k8s.cloudscale.ch/loadbalancer-health-monitor-timeout-s": "2", + }, ServicePortSpec{Protocol: v1.ProtocolUDP, Port: 5000, TargetPort: 5353}) + + // Wait for the service to be ready + s.T().Log("Waiting for udp-echo service to be ready") + service := s.AwaitServiceReady("udp-echo", 180*time.Second) + s.Require().NotNil(service) + + // Ensure the annotations are set + s.Assert().NotEmpty( + service.Annotations[cloudscale_ccm.LoadBalancerUUID]) + s.Assert().NotEmpty( + service.Annotations[cloudscale_ccm.LoadBalancerConfigVersion]) + s.Assert().NotEmpty( + service.Annotations[cloudscale_ccm.LoadBalancerZone]) + + // Ensure we have two public IP addresses + s.Require().Len(service.Status.LoadBalancer.Ingress, 2) + addr := service.Status.LoadBalancer.Ingress[0].IP + + // Verify UDP service responses using Go's UDP client + s.T().Log("Verifying UDP echo service responses") + errors := 0 + successes := 0 + + // Create UDP client + conn, err := net.Dial("udp", fmt.Sprintf("%s:5000", addr)) + s.Require().NoError(err) + s.T().Log("UDP client connected successfully") + defer conn.Close() + + // Set read timeout + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + + message := []byte("Tell me a joke") + for i := 0; i < 10; i++ { + // Send message + _, err := conn.Write(message) + if err != nil { + s.T().Logf("Failed to send message %d: %s", i, err) + errors++ + continue + } + + // Read response + buffer := make([]byte, 1024) + n, err := conn.Read(buffer) + if err != nil { + s.T().Logf("Failed to read response %d: %s", i, err) + errors++ + continue + } + + response := string(buffer[:n]) + if strings.Contains(response, "UDP joke") { + successes++ + } + + time.Sleep(250 * time.Millisecond) + } + + // Expect most requests to succeed + s.T().Logf("Successful probes: %d, Error probes: %d", successes, errors) + s.Assert().GreaterOrEqual(successes, 8) + + // In this simple case we expect no errors nor warnings + s.T().Log("Checking log output for errors/warnings") + lines := s.CCMLogs(start) + + s.Assert().NotContains(lines, "error") + s.Assert().NotContains(lines, "Error") + s.Assert().NotContains(lines, "warn") + s.Assert().NotContains(lines, "Warn") +} + +func (s *IntegrationTestSuite) TestServiceEndToEndDualProtocol() { + + // Note the start for the log + start := time.Now() + + // Deploy a DNS server that handles both TCP and UDP + s.T().Log("Creating dns-server deployment") + + // Create the ConfigMap for CoreDNS configuration + s.CreateConfigMap("coredns-config", map[string]string{ + "Corefile": `.:53 { + log + errors + health + ready + whoami + forward . 8.8.8.8 9.9.9.9 +}`, + }) + + // Create deployment with both TCP and UDP ports + replicas := int32(2) + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "dns-server"}, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "dns-server", + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "dns-server", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "coredns", + Image: "coredns/coredns:1.11.1", + Args: []string{"-conf", "/etc/coredns/Corefile"}, + Ports: []v1.ContainerPort{ + {ContainerPort: 53, Protocol: v1.ProtocolUDP, Name: "dns-udp"}, + {ContainerPort: 53, Protocol: v1.ProtocolTCP, Name: "dns-tcp"}, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "config", + MountPath: "/etc/coredns", + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "config", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "coredns-config", + }, + }, + }, + }, + }, + }, + }, + }, + } + + _, err := s.k8s.AppsV1().Deployments(s.ns).Create( + context.Background(), + deployment, + metav1.CreateOptions{}, + ) + s.Require().NoError(err) + + s.ExposeDeployment("dns-server", nil, + ServicePortSpec{Protocol: v1.ProtocolTCP, Port: 53, TargetPort: 53}, + ServicePortSpec{Protocol: v1.ProtocolUDP, Port: 53, TargetPort: 53}, + ) + + // Wait for the service to be ready + s.T().Log("Waiting for dns-server service to be ready") + svc := s.AwaitServiceReady("dns-server", 180*time.Second) + s.Require().NotNil(svc) + + // Ensure the annotations are set + s.Assert().NotEmpty( + svc.Annotations[cloudscale_ccm.LoadBalancerUUID]) + s.Assert().NotEmpty( + svc.Annotations[cloudscale_ccm.LoadBalancerConfigVersion]) + s.Assert().NotEmpty( + svc.Annotations[cloudscale_ccm.LoadBalancerZone]) + + // Ensure we have two public IP addresses + s.Require().Len(svc.Status.LoadBalancer.Ingress, 2) + addr := svc.Status.LoadBalancer.Ingress[0].IP + + // Create custom resolver pointing to the DNS service + resolver := &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + d := net.Dialer{Timeout: 5 * time.Second} + return d.DialContext(ctx, network, fmt.Sprintf("%s:53", addr)) + }, + } + + // Test UDP DNS queries (default) + s.T().Log("Verifying UDP DNS service responses") + udpSuccesses := 0 + udpErrors := 0 + + for i := 0; i < 10; i++ { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + mx, err := resolver.LookupMX(ctx, "cloudscale.ch") + cancel() + s.Require().NoError(err) + s.Require().Len(mx, 1) + s.Assert().Equal("mail.cloudscale.ch.", mx[0].Host) + + if err != nil { + s.T().Logf("UDP query %d failed: %s", i, err) + udpErrors++ + } else { + udpSuccesses++ + } + + time.Sleep(250 * time.Millisecond) + } + + s.T().Logf("UDP - Successful queries: %d, Errors: %d", udpSuccesses, udpErrors) + s.Assert().GreaterOrEqual(udpSuccesses, 8) + + // Test TCP DNS queries + s.T().Log("Verifying TCP DNS service responses") + tcpResolver := &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + d := net.Dialer{Timeout: 5 * time.Second} + // Force TCP by specifying "tcp" explicitly + return d.DialContext(ctx, "tcp", fmt.Sprintf("%s:53", addr)) + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + mx, err := tcpResolver.LookupMX(ctx, "cloudscale.ch") + cancel() + s.Require().NoError(err) + s.Require().Len(mx, 1) + s.Assert().Equal("mail.cloudscale.ch.", mx[0].Host) + + tcpSuccesses := 0 + + if err != nil { + s.T().Logf("TCP query failed: %s", err) + } else { + tcpSuccesses++ + } + + s.T().Logf("TCP - Successful queries: %d", tcpSuccesses) + s.Assert().Equal(tcpSuccesses, 1) + + // In this simple case we expect no errors nor warnings + s.T().Log("Checking log output for errors/warnings") + lines := s.CCMLogs(start) + + s.Assert().NotContains(lines, "error") + s.Assert().NotContains(lines, "Error") + s.Assert().NotContains(lines, "warn") + s.Assert().NotContains(lines, "Warn") +} + func (s *IntegrationTestSuite) TestServiceVIPAddresses() { // Get the private subnet used by the nodes @@ -358,13 +654,13 @@ func (s *IntegrationTestSuite) TestServiceVIPAddresses() { // Deploy a TCP server that returns something s.T().Log("Creating foo deployment") - s.CreateDeployment("nginx", "nginxdemos/hello:plain-text", 2, 80) + s.CreateDeployment("nginx", "nginxdemos/hello:plain-text", 2, v1.ProtocolTCP, 80) // Expose the deployment using a LoadBalancer service - s.ExposeDeployment("nginx", 80, 80, map[string]string{ + s.ExposeDeployment("nginx", map[string]string{ "k8s.cloudscale.ch/loadbalancer-vip-addresses": fmt.Sprintf( `[{"subnet": "%s"}]`, subnet), - }) + }, ServicePortSpec{Protocol: v1.ProtocolTCP, Port: 80, TargetPort: 80}) s.T().Log("Waiting for nginx service to be ready") service := s.AwaitServiceReady("nginx", 180*time.Second) @@ -398,7 +694,7 @@ func (s *IntegrationTestSuite) TestServiceTrafficPolicyLocal() { // single instance as we want to check that the routing works right with // all policies. s.T().Log("Creating peeraddr deployment") - s.CreateDeployment("peeraddr", "ghcr.io/majd/ip-curl", 1, 3000) + s.CreateDeployment("peeraddr", "ghcr.io/majd/ip-curl", 1, v1.ProtocolTCP, 3000) // Waits until the request is received through the given prefix and // ten responses with the expected address come back. @@ -460,7 +756,7 @@ func (s *IntegrationTestSuite) TestServiceTrafficPolicyLocal() { } // Expose the deployment using a LoadBalancer service - s.ExposeDeployment("peeraddr", 80, 3000, nil) + s.ExposeDeployment("peeraddr", nil, ServicePortSpec{Protocol: v1.ProtocolTCP, Port: 80, TargetPort: 3000}) // Wait for the service to be ready s.T().Log("Waiting for peeraddr service to be ready") @@ -524,13 +820,13 @@ func (s *IntegrationTestSuite) RunTestServiceWithFloatingIP( // Deploy a TCP server that returns the hostname s.T().Log("Creating nginx deployment") - s.CreateDeployment("nginx", "nginxdemos/hello:plain-text", 2, 80) + s.CreateDeployment("nginx", "nginxdemos/hello:plain-text", 2, v1.ProtocolTCP, 80) // Expose the deployment using a LoadBalancer service with Floating IP - s.ExposeDeployment("nginx", 80, 80, map[string]string{ + s.ExposeDeployment("nginx", map[string]string{ "k8s.cloudscale.ch/loadbalancer-floating-ips": fmt.Sprintf( `["%s"]`, fip.Network), - }) + }, ServicePortSpec{Protocol: v1.ProtocolTCP, Port: 80, TargetPort: 80}) // Wait for the service to be ready s.T().Log("Waiting for nginx service to be ready") @@ -586,13 +882,13 @@ func (s *IntegrationTestSuite) TestFloatingIPConflicts() { // Deploy a TCP server that returns the hostname s.T().Log("Creating nginx deployment") - s.CreateDeployment("nginx", "nginxdemos/hello:plain-text", 2, 80) + s.CreateDeployment("nginx", "nginxdemos/hello:plain-text", 2, v1.ProtocolTCP, 80) // Expose the deployment using a LoadBalancer service with Floating IP - s.ExposeDeployment("nginx", 80, 80, map[string]string{ + s.ExposeDeployment("nginx", map[string]string{ "k8s.cloudscale.ch/loadbalancer-floating-ips": fmt.Sprintf( `["%s"]`, regional.Network), - }) + }, ServicePortSpec{Protocol: v1.ProtocolTCP, Port: 80, TargetPort: 80}) // Wait for the service to be ready s.T().Log("Waiting for nginx service to be ready") @@ -602,10 +898,10 @@ func (s *IntegrationTestSuite) TestFloatingIPConflicts() { // Configure a second service with the same floating IP start := time.Now() - s.ExposeDeployment("service-2", 80, 80, map[string]string{ + s.ExposeDeployment("service-2", map[string]string{ "k8s.cloudscale.ch/loadbalancer-floating-ips": fmt.Sprintf( `["%s"]`, regional.Network), - }) + }, ServicePortSpec{Protocol: v1.ProtocolTCP, Port: 80, TargetPort: 80}) // Wait for a moment before checking the log time.Sleep(5 * time.Second) @@ -626,7 +922,7 @@ func (s *IntegrationTestSuite) TestServiceProxyProtocol() { // Deploy our http-echo server to check for proxy connections s.T().Log("Creating http-echo deployment", "branch", branch) - s.CreateDeployment("http-echo", "golang", 2, 80, "bash", "-c", fmt.Sprintf(` + s.CreateDeployment("http-echo", "golang", 2, v1.ProtocolTCP, 80, "bash", "-c", fmt.Sprintf(` git clone https://github.com/cloudscale-ch/cloudscale-cloud-controller-manager ccm; cd ccm; git checkout %s || exit 1; @@ -635,13 +931,13 @@ func (s *IntegrationTestSuite) TestServiceProxyProtocol() { `, branch)) // Expose the deployment using a LoadBalancer service - s.ExposeDeployment("http-echo", 80, 80, map[string]string{ + s.ExposeDeployment("http-echo", map[string]string{ "k8s.cloudscale.ch/loadbalancer-pool-protocol": "proxy", // Make sure to get the default behavior of older Kubernetes releases, // even on newer releases. "k8s.cloudscale.ch/loadbalancer-ip-mode": "VIP", - }) + }, ServicePortSpec{Protocol: v1.ProtocolTCP, Port: 80, TargetPort: 80}) // Wait for the service to be ready s.T().Log("Waiting for http-echo service to be ready") @@ -682,14 +978,14 @@ func (s *IntegrationTestSuite) TestServiceProxyProtocol() { s.Assert().Equal("false\n", used) // The workaround works by using an IP that needs to be reolved via name - s.ExposeDeployment("http-echo", 80, 80, map[string]string{ + s.ExposeDeployment("http-echo", map[string]string{ "k8s.cloudscale.ch/loadbalancer-pool-protocol": "proxy", "k8s.cloudscale.ch/loadbalancer-ip-mode": "VIP", "k8s.cloudscale.ch/loadbalancer-force-hostname": fmt.Sprintf( "%s.cust.cloudscale.ch", strings.ReplaceAll(addr, ".", "-"), ), - }) + }, ServicePortSpec{Protocol: v1.ProtocolTCP, Port: 80, TargetPort: 80}) s.T().Log("Testing PROXY protocol from inside with workaround") used = s.RunJob("curlimages/curl", 90*time.Second, "curl", "-s", url) @@ -700,9 +996,9 @@ func (s *IntegrationTestSuite) TestServiceProxyProtocol() { s.Assert().NoError(err) if newer { - s.ExposeDeployment("http-echo", 80, 80, map[string]string{ + s.ExposeDeployment("http-echo", map[string]string{ "k8s.cloudscale.ch/loadbalancer-pool-protocol": "proxy", - }) + }, ServicePortSpec{Protocol: v1.ProtocolTCP, Port: 80, TargetPort: 80}) s.T().Log("Testing PROXY protocol on newer Kubernetes releases") used = s.RunJob("curlimages/curl", 90*time.Second, "curl", "-s", url) From 5795d13a0c9492f55a5c63f28f7ae4ec9a0714df Mon Sep 17 00:00:00 2001 From: Alain Kaeslin Date: Fri, 31 Oct 2025 11:45:58 +0100 Subject: [PATCH 4/7] Add health probes and remove load balancer annotations in dns-dual-protocol example. --- examples/dns-dual-protocol.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/examples/dns-dual-protocol.yaml b/examples/dns-dual-protocol.yaml index f69f476..21b97c4 100644 --- a/examples/dns-dual-protocol.yaml +++ b/examples/dns-dual-protocol.yaml @@ -65,6 +65,14 @@ spec: - containerPort: 53 protocol: TCP name: dns-tcp + livenessProbe: + httpGet: + path: /health + port: 8080 + readinessProbe: + httpGet: + path: /ready + port: 8181 volumes: - name: config configMap: @@ -73,9 +81,6 @@ spec: apiVersion: v1 kind: Service metadata: - annotations: - k8s.cloudscale.ch/loadbalancer-health-monitor-delay-s: "3" - k8s.cloudscale.ch/loadbalancer-health-monitor-timeout-s: "2" labels: app: dns-server name: dns-server From f74dc156b44631b7c4a392c29ae195bd89481a9c Mon Sep 17 00:00:00 2001 From: Alain Kaeslin Date: Fri, 31 Oct 2025 11:49:56 +0100 Subject: [PATCH 5/7] Fix typo in load balancer protocol documentation. --- pkg/cloudscale_ccm/loadbalancer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cloudscale_ccm/loadbalancer.go b/pkg/cloudscale_ccm/loadbalancer.go index 4cb8dc1..82ee99c 100644 --- a/pkg/cloudscale_ccm/loadbalancer.go +++ b/pkg/cloudscale_ccm/loadbalancer.go @@ -119,7 +119,7 @@ const ( // with protocol `TCP` use. Set it to `proxy` and `proxyv2` if TCP // traffic should be forwarded using these protocols. // - // When setting the protocol of a port to `UDP,` traffic is always forwarded + // When setting the protocol of a port to `UDP`, traffic is always forwarded // using UDP. // // We are technically able to have different protocols for From 0ebf7c6469c84922815cd647e54425c84e4927e5 Mon Sep 17 00:00:00 2001 From: Alain Kaeslin Date: Fri, 31 Oct 2025 13:54:09 +0100 Subject: [PATCH 6/7] Add delay for UDP service test stability. --- pkg/internal/integration/service_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/internal/integration/service_test.go b/pkg/internal/integration/service_test.go index d19befc..954bf1f 100644 --- a/pkg/internal/integration/service_test.go +++ b/pkg/internal/integration/service_test.go @@ -404,6 +404,13 @@ func (s *IntegrationTestSuite) TestServiceEndToEndUDP() { s.Require().Len(service.Status.LoadBalancer.Ingress, 2) addr := service.Status.LoadBalancer.Ingress[0].IP + // We have to wait a few seconds until the configured udp-connect + // health monitor reports the services as up. Depending on the + // timings, we could see i/o timeouts otherwise. + // Sleeping 9 seconds allows for at least 3 successful up probes, + // which is one more than required. + time.Sleep(3 * 3 * time.Second) + // Verify UDP service responses using Go's UDP client s.T().Log("Verifying UDP echo service responses") errors := 0 @@ -412,7 +419,7 @@ func (s *IntegrationTestSuite) TestServiceEndToEndUDP() { // Create UDP client conn, err := net.Dial("udp", fmt.Sprintf("%s:5000", addr)) s.Require().NoError(err) - s.T().Log("UDP client connected successfully") + s.T().Log("UDP client created successfully") defer conn.Close() // Set read timeout From 47b40428c95803728a6c47cae7f51417c96995b6 Mon Sep 17 00:00:00 2001 From: Alain Kaeslin Date: Fri, 31 Oct 2025 13:55:56 +0100 Subject: [PATCH 7/7] Update coredns image version in test. --- pkg/internal/integration/service_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/internal/integration/service_test.go b/pkg/internal/integration/service_test.go index 954bf1f..fb7f1d3 100644 --- a/pkg/internal/integration/service_test.go +++ b/pkg/internal/integration/service_test.go @@ -507,7 +507,7 @@ func (s *IntegrationTestSuite) TestServiceEndToEndDualProtocol() { Containers: []v1.Container{ { Name: "coredns", - Image: "coredns/coredns:1.11.1", + Image: "docker.io/coredns/coredns:1.13.1", Args: []string{"-conf", "/etc/coredns/Corefile"}, Ports: []v1.ContainerPort{ {ContainerPort: 53, Protocol: v1.ProtocolUDP, Name: "dns-udp"},