Configure CoreDNS and kubelet for dual stack DNS

cmd/controller/controller.go

@@ -16,7 +16,8 @@
 	"os"
 	"path/filepath"
 	"slices"
 	"sync/atomic"
+	"strings"
 	"time"
 
 	"github.com/k0sproject/k0s/cmd/internal"
@@ -241,11 +242,15 @@
 	logrus.Infof("using listen port: %d", nodeConfig.Spec.API.Port)
 	logrus.Infof("using sans: %s", nodeConfig.Spec.API.SANs)
 
-	dnsAddress, err := nodeConfig.Spec.Network.DNSAddress(nodeConfig.Spec.PrimaryAddressFamily())
+	dnsAddresses, err := nodeConfig.Spec.Network.DNSAddresses(nodeConfig.Spec.PrimaryAddressFamily())
 	if err != nil {
 		return err
 	}
-	logrus.Infof("DNS address: %s", dnsAddress)
+	addressStrings := make([]string, len(dnsAddresses))
+	for i, address := range dnsAddresses {
+		addressStrings[i] = address.String()
+	}
+	logrus.Infof("DNS addresses: %s", strings.Join(addressStrings, ", "))
 
 	var storageBackend manager.Component
 	var leaveEtcdClusterOnStop *atomic.Bool

pkg/apis/k0s/v1beta1/network.go

@@ -187,32 +187,44 @@ func (n *Network) Validate() []error {
 	return errors
 }
 
-// DNSAddress calculates the 10th address of configured service CIDR block.
-func (n *Network) DNSAddress(primaryAddressFamily PrimaryAddressFamilyType) (string, error) {
-	serviceCIDR := n.ServiceCIDR
-	if n.DualStack.Enabled && primaryAddressFamily == PrimaryFamilyIPv6 {
-		serviceCIDR = n.DualStack.IPv6ServiceCIDR
+func (n *Network) DNSAddresses(primaryAddressFamily PrimaryAddressFamilyType) ([]net.IP, error) {
+	var cidrs []string
+	if n.DualStack.Enabled {
+		if primaryAddressFamily == PrimaryFamilyIPv6 {
+			cidrs = []string{n.DualStack.IPv6ServiceCIDR, n.ServiceCIDR}
+		} else {
+			cidrs = []string{n.ServiceCIDR, n.DualStack.IPv6ServiceCIDR}
+		}
+	} else {
+		cidrs = []string{n.ServiceCIDR}
 	}
 
-	_, ipnet, err := net.ParseCIDR(serviceCIDR)
-	if err != nil {
-		return "", fmt.Errorf("failed to parse service CIDR %q: %w", serviceCIDR, err)
-	}
+	var addresses []net.IP
 
-	addr := slices.Clone(ipnet.IP)
+	for _, cidr := range cidrs {
 
-	maskLen, netLen := ipnet.Mask.Size()
-	if netLen-maskLen > 3 {
-		addr[len(addr)-1] += 10
-	} else {
-		addr[len(addr)-1] += 2
-	}
+		_, ipnet, err := net.ParseCIDR(cidr)
+		if err != nil {
+			return []net.IP{}, fmt.Errorf("failed to parse service CIDR %q: %w", cidr, err)
+		}
+
+		addr := slices.Clone(ipnet.IP)
+
+		maskLen, netLen := ipnet.Mask.Size()
+		if netLen-maskLen > 3 {
+			addr[len(addr)-1] += 10
+		} else {
+			addr[len(addr)-1] += 2
+		}
+
+		if !ipnet.Contains(addr) {
+			return []net.IP{}, fmt.Errorf("failed to calculate DNS address: CIDR too narrow: %s", cidr)
+		}
 
-	if !ipnet.Contains(addr) {
-		return "", fmt.Errorf("failed to calculate DNS address: CIDR too narrow: %s", n.ServiceCIDR)
+		addresses = append(addresses, addr)
 	}
 
-	return addr.String(), nil
+	return addresses, nil
 }
 
 // InternalAPIAddresses calculates the internal API address of configured service CIDR block.

pkg/apis/k0s/v1beta1/network_test.go

@@ -4,6 +4,7 @@
 package v1beta1
 
 import (
+	"net"
 	"testing"
 
 	"k8s.io/utils/ptr"
@@ -16,41 +17,49 @@ type NetworkSuite struct {
 	suite.Suite
 }
 
+func ipsAsStrings(ips []net.IP) []string {
+	strings := make([]string, len(ips))
+	for i, ip := range ips {
+		strings[i] = ip.String()
+	}
+	return strings
+}
+
 func (s *NetworkSuite) TestAddresses() {
 	s.Run("DNS_default_service_cidr", func() {
 		n := DefaultNetwork()
-		dns, err := n.DNSAddress(PrimaryFamilyIPv4)
+		dns, err := n.DNSAddresses(PrimaryFamilyIPv4)
 		s.Require().NoError(err)
-		s.Equal("10.96.0.10", dns)
+		s.Equal([]string{"10.96.0.10"}, ipsAsStrings(dns))
 	})
 	s.Run("DNS_uses_non_default_service_cidr", func() {
 		n := DefaultNetwork()
 		n.ServiceCIDR = "10.96.0.248/29"
-		dns, err := n.DNSAddress(PrimaryFamilyIPv4)
+		dns, err := n.DNSAddresses(PrimaryFamilyIPv4)
 		s.Require().NoError(err)
-		s.Equal("10.96.0.250", dns)
+		s.Equal([]string{"10.96.0.250"}, ipsAsStrings(dns))
 	})
 	s.Run("DNS_service_cidr_too_narrow", func() {
 		n := Network{ServiceCIDR: "192.168.178.0/31"}
-		dns, err := n.DNSAddress(PrimaryFamilyIPv4)
+		dns, err := n.DNSAddresses(PrimaryFamilyIPv4)
 		s.Empty(dns)
 		s.ErrorContains(err, "failed to calculate DNS address: CIDR too narrow: 192.168.178.0/31")
 	})
 	s.Run("DNS_uses_v6_service_cidr", func() {
 		n := Network{ServiceCIDR: "fd00:abcd:1234::/64"}
-		dns, err := n.DNSAddress(PrimaryFamilyIPv6)
+		dns, err := n.DNSAddresses(PrimaryFamilyIPv6)
 		s.NoError(err)
-		s.Equal("fd00:abcd:1234::a", dns)
+		s.Equal([]string{"fd00:abcd:1234::a"}, ipsAsStrings(dns))
 	})
 	s.Run("DNS_uses_v6_small_service_cidr", func() {
 		n := Network{ServiceCIDR: "fd00::/126"}
-		dns, err := n.DNSAddress(PrimaryFamilyIPv6)
+		dns, err := n.DNSAddresses(PrimaryFamilyIPv6)
 		s.NoError(err)
-		s.Equal("fd00::2", dns)
+		s.Equal([]string{"fd00::2"}, ipsAsStrings(dns))
 	})
 	s.Run("DNS_service_v6_cidr_too_narrow", func() {
 		n := Network{ServiceCIDR: "fd00::/127"}
-		dns, err := n.DNSAddress(PrimaryFamilyIPv6)
+		dns, err := n.DNSAddresses(PrimaryFamilyIPv6)
 		s.Empty(dns)
 		s.ErrorContains(err, "failed to calculate DNS address: CIDR too narrow: fd00::/127")
 	})
@@ -95,6 +104,30 @@ func (s *NetworkSuite) TestAddresses() {
 			s.Equal(n.DualStack.IPv6ServiceCIDR+","+n.ServiceCIDR, n.BuildServiceCIDR(PrimaryFamilyIPv6))
 		})
 	})
+	s.Run("DNS_uses_dual_stack_service_cidrs_primary_ipv4", func() {
+		n := Network{
+			ServiceCIDR: "10.96.0.248/29",
+			DualStack: DualStack{
+				Enabled:         true,
+				IPv6ServiceCIDR: "fd00:abcd:1234::/64",
+			},
+		}
+		dns, err := n.DNSAddresses(PrimaryFamilyIPv4)
+		s.NoError(err)
+		s.Equal([]string{"10.96.0.250", "fd00:abcd:1234::a"}, ipsAsStrings(dns))
+	})
+	s.Run("DNS_uses_dual_stack_service_cidrs_primary_ipv6", func() {
+		n := Network{
+			ServiceCIDR: "10.96.0.248/29",
+			DualStack: DualStack{
+				Enabled:         true,
+				IPv6ServiceCIDR: "fd00:abcd:1234::/64",
+			},
+		}
+		dns, err := n.DNSAddresses(PrimaryFamilyIPv6)
+		s.NoError(err)
+		s.Equal([]string{"fd00:abcd:1234::a", "10.96.0.250"}, ipsAsStrings(dns))
+	})
 }
 
 func (s *NetworkSuite) TestDomainMarshaling() {

pkg/component/controller/calico.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
+	"net"
 	"path"
 	"path/filepath"
 	"strings"
@@ -73,7 +74,7 @@ type calicoConfig struct {
 type calicoNodeConfig struct {
 	APIServer       *k0snet.HostPort
 	ServiceCIDRIPv4 string
-	ClusterDNSIP    string
+	ClusterDNSIPs   []net.IP
 }
 
 type calicoClusterConfig struct {
@@ -102,7 +103,7 @@ type calicoClusterConfig struct {
 
 // NewCalico creates new Calico reconciler component
 func NewCalico(nodeConfig *v1beta1.ClusterConfig, manifestsDir string, hasWindowsNodes func() (*bool, <-chan struct{})) (*Calico, error) {
-	dnsAddress, err := nodeConfig.Spec.Network.DNSAddress(nodeConfig.Spec.PrimaryAddressFamily())
+	dnsAddresses, err := nodeConfig.Spec.Network.DNSAddresses(nodeConfig.Spec.PrimaryAddressFamily())
 	if err != nil {
 		return nil, err
 	}
@@ -117,7 +118,7 @@ func NewCalico(nodeConfig *v1beta1.ClusterConfig, manifestsDir string, hasWindow
 		nodeConfig: calicoNodeConfig{
 			APIServer:       apiServer,
 			ServiceCIDRIPv4: nodeConfig.Spec.Network.ServiceCIDR,
-			ClusterDNSIP:    dnsAddress,
+			ClusterDNSIPs:   dnsAddresses,
 		},
 		primaryAddressFamily: nodeConfig.Spec.PrimaryAddressFamily(),
 		manifestsDir:         manifestsDir,

pkg/component/controller/coredns.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 	"math"
+	"net"
 	"path/filepath"
 	"reflect"
 	"time"
@@ -250,7 +251,18 @@ metadata:
 spec:
   selector:
     k8s-app: kube-dns
-  clusterIP: {{ .ClusterDNSIP }}
+  clusterIP: {{ index .ClusterDNSIPs 0 }}
+  {{- if .DualStack }}
+  clusterIPs:
+  {{- range .ClusterDNSIPs }}
+  - {{ . | quote }}
+  {{- end }}
+  ipFamilies:
+  {{- range .ClusterDNSIPFamilies }}
+  - {{ . | quote }}
+  {{- end }}
+  ipFamilyPolicy: PreferDualStack
+  {{- end }}
   ports:
   - name: dns
     port: 53
@@ -270,7 +282,7 @@ var _ manager.Reconciler = (*CoreDNS)(nil)
 
 // CoreDNS is the component implementation to manage CoreDNS
 type CoreDNS struct {
-	dnsAddress             string
+	dnsAddresses           []net.IP
 	clusterDomain          string
 	client                 metadata.Interface
 	log                    *logrus.Entry
@@ -283,18 +295,20 @@ type CoreDNS struct {
 
 type coreDNSConfig struct {
 	Replicas                   int
-	ClusterDNSIP               string
+	ClusterDNSIPs              []string
+	ClusterDNSIPFamilies       []string
 	ClusterDomain              string
 	Image                      string
 	PullPolicy                 string
 	MaxUnavailableReplicas     *uint
 	DisablePodAntiAffinity     bool
 	DisablePodDisruptionBudget bool
+	DualStack                  bool
 }
 
 // NewCoreDNS creates new instance of CoreDNS component
 func NewCoreDNS(k0sVars *config.CfgVars, clientFactory k8sutil.ClientFactoryInterface, nodeConfig *v1beta1.ClusterConfig) (*CoreDNS, error) {
-	dnsAddress, err := nodeConfig.Spec.Network.DNSAddress(nodeConfig.Spec.PrimaryAddressFamily())
+	dnsAddresses, err := nodeConfig.Spec.Network.DNSAddresses(nodeConfig.Spec.PrimaryAddressFamily())
 	if err != nil {
 		return nil, err
 	}
@@ -310,7 +324,7 @@ func NewCoreDNS(k0sVars *config.CfgVars, clientFactory k8sutil.ClientFactoryInte
 	}
 
 	return &CoreDNS{
-		dnsAddress:    dnsAddress,
+		dnsAddresses:  dnsAddresses,
 		clusterDomain: nodeConfig.Spec.Network.ClusterDomain,
 		client:        client,
 		log:           logrus.WithField("component", "coredns"),
@@ -361,12 +375,28 @@ func (c *CoreDNS) getConfig(ctx context.Context, clusterConfig *v1beta1.ClusterC
 
 	nodeCount := len(nodes.Items)
 
+	var addresses []string
+	var addressFamilies []string
+	for _, address := range c.dnsAddresses {
+		addresses = append(addresses, address.String())
+		if address == nil {
+			return coreDNSConfig{}, fmt.Errorf("invalid IP address %q", address)
+		}
+		if address.To4() == nil {
+			addressFamilies = append(addressFamilies, "IPv6")
+		} else {
+			addressFamilies = append(addressFamilies, "IPv4")
+		}
+	}
+
 	config := coreDNSConfig{
-		Replicas:      replicaCount(nodeCount),
-		ClusterDomain: c.clusterDomain,
-		ClusterDNSIP:  c.dnsAddress,
-		Image:         clusterConfig.Spec.Images.CoreDNS.URI(),
-		PullPolicy:    clusterConfig.Spec.Images.DefaultPullPolicy,
+		Replicas:             replicaCount(nodeCount),
+		ClusterDomain:        c.clusterDomain,
+		ClusterDNSIPs:        addresses,
+		ClusterDNSIPFamilies: addressFamilies,
+		Image:                clusterConfig.Spec.Images.CoreDNS.URI(),
+		PullPolicy:           clusterConfig.Spec.Images.DefaultPullPolicy,
+		DualStack:            clusterConfig.Spec.Network.DualStack.Enabled,
 	}
 
 	if config.Replicas <= 1 {

pkg/component/controller/coredns_test.go

@@ -18,7 +18,7 @@ func TestCoreDNS_RenderWithPatch(t *testing.T) {
 	cfg := coreDNSConfig{
 		Replicas:      1,
 		ClusterDomain: "cluster.local",
-		ClusterDNSIP:  "10.96.0.10",
+		ClusterDNSIPs: []string{"10.96.0.10"},
 		Image:         "coredns:latest",
 		PullPolicy:    "IfNotPresent",
 	}

pkg/component/controller/workerconfig/reconciler.go

@@ -52,7 +52,7 @@ type Reconciler struct {
 	log logrus.FieldLogger
 
 	clusterDomain        string
-	clusterDNSIP         net.IP
+	clusterDNSIPs        []net.IP
 	clientFactory        kubeutil.ClientFactoryInterface
 	leaderElector        leaderelector.Interface
 	konnectivityEnabled  bool
@@ -89,20 +89,16 @@ var (
 func NewReconciler(k0sVars *config.CfgVars, nodeConfig *v1beta1.ClusterConfig, clientFactory kubeutil.ClientFactoryInterface, leaderElector leaderelector.Interface, konnectivityEnabled, autopilotDisabled bool) (*Reconciler, error) {
 	log := logrus.WithFields(logrus.Fields{"component": "workerconfig.Reconciler"})
 
-	clusterDNSIPString, err := nodeConfig.Spec.Network.DNSAddress(nodeConfig.Spec.PrimaryAddressFamily())
+	clusterDNSIPs, err := nodeConfig.Spec.Network.DNSAddresses(nodeConfig.Spec.PrimaryAddressFamily())
 	if err != nil {
 		return nil, err
 	}
-	clusterDNSIP := net.ParseIP(clusterDNSIPString)
-	if clusterDNSIP == nil {
-		return nil, fmt.Errorf("not an IP address: %q", clusterDNSIPString)
-	}
 
 	reconciler := &Reconciler{
 		log: log,
 
 		clusterDomain:        nodeConfig.Spec.Network.ClusterDomain,
-		clusterDNSIP:         clusterDNSIP,
+		clusterDNSIPs:        clusterDNSIPs,
 		clientFactory:        clientFactory,
 		leaderElector:        leaderElector,
 		konnectivityEnabled:  konnectivityEnabled,
@@ -579,6 +575,10 @@ func (r *Reconciler) buildProfile(snapshot *snapshot) *workerconfig.Profile {
 	for i, cipherSuite := range constant.AllowedTLS12CipherSuiteIDs {
 		cipherSuites[i] = tls.CipherSuiteName(cipherSuite)
 	}
+	var clusterDNSIPs []string
+	for _, ip := range r.clusterDNSIPs {
+		clusterDNSIPs = append(clusterDNSIPs, ip.String())
+	}
 
 	workerProfile := &workerconfig.Profile{
 		APIServerAddresses: slices.Clone(snapshot.apiServers),
@@ -588,7 +588,7 @@ func (r *Reconciler) buildProfile(snapshot *snapshot) *workerconfig.Profile {
 				APIVersion: kubeletv1beta1.SchemeGroupVersion.String(),
 				Kind:       "KubeletConfiguration",
 			},
-			ClusterDNS:         []string{r.clusterDNSIP.String()},
+			ClusterDNS:         clusterDNSIPs,
 			ClusterDomain:      r.clusterDomain,
 			KubeReservedCgroup: "system.slice",
 			KubeletCgroups:     "/system.slice/containerd.service",