diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go index 51a3e0b45eac..604c54de6c47 100644 --- a/cmd/controller/controller.go +++ b/cmd/controller/controller.go @@ -17,6 +17,7 @@ import ( "path/filepath" "slices" "sync/atomic" + "strings" "time" "github.com/k0sproject/k0s/cmd/internal" @@ -241,11 +242,15 @@ func (c *command) start(ctx context.Context, rtc *config.RuntimeConfig, nodeConf logrus.Infof("using listen port: %d", nodeConfig.Spec.API.Port) logrus.Infof("using sans: %s", nodeConfig.Spec.API.SANs) - dnsAddress, err := nodeConfig.Spec.Network.DNSAddress(nodeConfig.Spec.PrimaryAddressFamily()) + dnsAddresses, err := nodeConfig.Spec.Network.DNSAddresses(nodeConfig.Spec.PrimaryAddressFamily()) if err != nil { return err } - logrus.Infof("DNS address: %s", dnsAddress) + addressStrings := make([]string, len(dnsAddresses)) + for i, address := range dnsAddresses { + addressStrings[i] = address.String() + } + logrus.Infof("DNS addresses: %s", strings.Join(addressStrings, ", ")) var storageBackend manager.Component var leaveEtcdClusterOnStop *atomic.Bool diff --git a/pkg/apis/k0s/v1beta1/network.go b/pkg/apis/k0s/v1beta1/network.go index 7023c9f6c08d..2da9f46fad50 100644 --- a/pkg/apis/k0s/v1beta1/network.go +++ b/pkg/apis/k0s/v1beta1/network.go @@ -187,32 +187,44 @@ func (n *Network) Validate() []error { return errors } -// DNSAddress calculates the 10th address of configured service CIDR block. -func (n *Network) DNSAddress(primaryAddressFamily PrimaryAddressFamilyType) (string, error) { - serviceCIDR := n.ServiceCIDR - if n.DualStack.Enabled && primaryAddressFamily == PrimaryFamilyIPv6 { - serviceCIDR = n.DualStack.IPv6ServiceCIDR +func (n *Network) DNSAddresses(primaryAddressFamily PrimaryAddressFamilyType) ([]net.IP, error) { + var cidrs []string + if n.DualStack.Enabled { + if primaryAddressFamily == PrimaryFamilyIPv6 { + cidrs = []string{n.DualStack.IPv6ServiceCIDR, n.ServiceCIDR} + } else { + cidrs = []string{n.ServiceCIDR, n.DualStack.IPv6ServiceCIDR} + } + } else { + cidrs = []string{n.ServiceCIDR} } - _, ipnet, err := net.ParseCIDR(serviceCIDR) - if err != nil { - return "", fmt.Errorf("failed to parse service CIDR %q: %w", serviceCIDR, err) - } + var addresses []net.IP - addr := slices.Clone(ipnet.IP) + for _, cidr := range cidrs { - maskLen, netLen := ipnet.Mask.Size() - if netLen-maskLen > 3 { - addr[len(addr)-1] += 10 - } else { - addr[len(addr)-1] += 2 - } + _, ipnet, err := net.ParseCIDR(cidr) + if err != nil { + return []net.IP{}, fmt.Errorf("failed to parse service CIDR %q: %w", cidr, err) + } + + addr := slices.Clone(ipnet.IP) + + maskLen, netLen := ipnet.Mask.Size() + if netLen-maskLen > 3 { + addr[len(addr)-1] += 10 + } else { + addr[len(addr)-1] += 2 + } + + if !ipnet.Contains(addr) { + return []net.IP{}, fmt.Errorf("failed to calculate DNS address: CIDR too narrow: %s", cidr) + } - if !ipnet.Contains(addr) { - return "", fmt.Errorf("failed to calculate DNS address: CIDR too narrow: %s", n.ServiceCIDR) + addresses = append(addresses, addr) } - return addr.String(), nil + return addresses, nil } // InternalAPIAddresses calculates the internal API address of configured service CIDR block. diff --git a/pkg/apis/k0s/v1beta1/network_test.go b/pkg/apis/k0s/v1beta1/network_test.go index 341e4694ece0..8d2683e1442d 100644 --- a/pkg/apis/k0s/v1beta1/network_test.go +++ b/pkg/apis/k0s/v1beta1/network_test.go @@ -4,6 +4,7 @@ package v1beta1 import ( + "net" "testing" "k8s.io/utils/ptr" @@ -16,41 +17,49 @@ type NetworkSuite struct { suite.Suite } +func ipsAsStrings(ips []net.IP) []string { + strings := make([]string, len(ips)) + for i, ip := range ips { + strings[i] = ip.String() + } + return strings +} + func (s *NetworkSuite) TestAddresses() { s.Run("DNS_default_service_cidr", func() { n := DefaultNetwork() - dns, err := n.DNSAddress(PrimaryFamilyIPv4) + dns, err := n.DNSAddresses(PrimaryFamilyIPv4) s.Require().NoError(err) - s.Equal("10.96.0.10", dns) + s.Equal([]string{"10.96.0.10"}, ipsAsStrings(dns)) }) s.Run("DNS_uses_non_default_service_cidr", func() { n := DefaultNetwork() n.ServiceCIDR = "10.96.0.248/29" - dns, err := n.DNSAddress(PrimaryFamilyIPv4) + dns, err := n.DNSAddresses(PrimaryFamilyIPv4) s.Require().NoError(err) - s.Equal("10.96.0.250", dns) + s.Equal([]string{"10.96.0.250"}, ipsAsStrings(dns)) }) s.Run("DNS_service_cidr_too_narrow", func() { n := Network{ServiceCIDR: "192.168.178.0/31"} - dns, err := n.DNSAddress(PrimaryFamilyIPv4) + dns, err := n.DNSAddresses(PrimaryFamilyIPv4) s.Empty(dns) s.ErrorContains(err, "failed to calculate DNS address: CIDR too narrow: 192.168.178.0/31") }) s.Run("DNS_uses_v6_service_cidr", func() { n := Network{ServiceCIDR: "fd00:abcd:1234::/64"} - dns, err := n.DNSAddress(PrimaryFamilyIPv6) + dns, err := n.DNSAddresses(PrimaryFamilyIPv6) s.NoError(err) - s.Equal("fd00:abcd:1234::a", dns) + s.Equal([]string{"fd00:abcd:1234::a"}, ipsAsStrings(dns)) }) s.Run("DNS_uses_v6_small_service_cidr", func() { n := Network{ServiceCIDR: "fd00::/126"} - dns, err := n.DNSAddress(PrimaryFamilyIPv6) + dns, err := n.DNSAddresses(PrimaryFamilyIPv6) s.NoError(err) - s.Equal("fd00::2", dns) + s.Equal([]string{"fd00::2"}, ipsAsStrings(dns)) }) s.Run("DNS_service_v6_cidr_too_narrow", func() { n := Network{ServiceCIDR: "fd00::/127"} - dns, err := n.DNSAddress(PrimaryFamilyIPv6) + dns, err := n.DNSAddresses(PrimaryFamilyIPv6) s.Empty(dns) s.ErrorContains(err, "failed to calculate DNS address: CIDR too narrow: fd00::/127") }) @@ -95,6 +104,30 @@ func (s *NetworkSuite) TestAddresses() { s.Equal(n.DualStack.IPv6ServiceCIDR+","+n.ServiceCIDR, n.BuildServiceCIDR(PrimaryFamilyIPv6)) }) }) + s.Run("DNS_uses_dual_stack_service_cidrs_primary_ipv4", func() { + n := Network{ + ServiceCIDR: "10.96.0.248/29", + DualStack: DualStack{ + Enabled: true, + IPv6ServiceCIDR: "fd00:abcd:1234::/64", + }, + } + dns, err := n.DNSAddresses(PrimaryFamilyIPv4) + s.NoError(err) + s.Equal([]string{"10.96.0.250", "fd00:abcd:1234::a"}, ipsAsStrings(dns)) + }) + s.Run("DNS_uses_dual_stack_service_cidrs_primary_ipv6", func() { + n := Network{ + ServiceCIDR: "10.96.0.248/29", + DualStack: DualStack{ + Enabled: true, + IPv6ServiceCIDR: "fd00:abcd:1234::/64", + }, + } + dns, err := n.DNSAddresses(PrimaryFamilyIPv6) + s.NoError(err) + s.Equal([]string{"fd00:abcd:1234::a", "10.96.0.250"}, ipsAsStrings(dns)) + }) } func (s *NetworkSuite) TestDomainMarshaling() { diff --git a/pkg/component/controller/calico.go b/pkg/component/controller/calico.go index 2e821daf7c53..8b1c65a20cf3 100644 --- a/pkg/component/controller/calico.go +++ b/pkg/component/controller/calico.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io/fs" + "net" "path" "path/filepath" "strings" @@ -73,7 +74,7 @@ type calicoConfig struct { type calicoNodeConfig struct { APIServer *k0snet.HostPort ServiceCIDRIPv4 string - ClusterDNSIP string + ClusterDNSIPs []net.IP } type calicoClusterConfig struct { @@ -102,7 +103,7 @@ type calicoClusterConfig struct { // NewCalico creates new Calico reconciler component func NewCalico(nodeConfig *v1beta1.ClusterConfig, manifestsDir string, hasWindowsNodes func() (*bool, <-chan struct{})) (*Calico, error) { - dnsAddress, err := nodeConfig.Spec.Network.DNSAddress(nodeConfig.Spec.PrimaryAddressFamily()) + dnsAddresses, err := nodeConfig.Spec.Network.DNSAddresses(nodeConfig.Spec.PrimaryAddressFamily()) if err != nil { return nil, err } @@ -117,7 +118,7 @@ func NewCalico(nodeConfig *v1beta1.ClusterConfig, manifestsDir string, hasWindow nodeConfig: calicoNodeConfig{ APIServer: apiServer, ServiceCIDRIPv4: nodeConfig.Spec.Network.ServiceCIDR, - ClusterDNSIP: dnsAddress, + ClusterDNSIPs: dnsAddresses, }, primaryAddressFamily: nodeConfig.Spec.PrimaryAddressFamily(), manifestsDir: manifestsDir, diff --git a/pkg/component/controller/coredns.go b/pkg/component/controller/coredns.go index 4b8e9a7da27b..d89cce6dccbb 100644 --- a/pkg/component/controller/coredns.go +++ b/pkg/component/controller/coredns.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "math" + "net" "path/filepath" "reflect" "time" @@ -250,7 +251,18 @@ metadata: spec: selector: k8s-app: kube-dns - clusterIP: {{ .ClusterDNSIP }} + clusterIP: {{ index .ClusterDNSIPs 0 }} + {{- if .DualStack }} + clusterIPs: + {{- range .ClusterDNSIPs }} + - {{ . | quote }} + {{- end }} + ipFamilies: + {{- range .ClusterDNSIPFamilies }} + - {{ . | quote }} + {{- end }} + ipFamilyPolicy: PreferDualStack + {{- end }} ports: - name: dns port: 53 @@ -270,7 +282,7 @@ var _ manager.Reconciler = (*CoreDNS)(nil) // CoreDNS is the component implementation to manage CoreDNS type CoreDNS struct { - dnsAddress string + dnsAddresses []net.IP clusterDomain string client metadata.Interface log *logrus.Entry @@ -283,18 +295,20 @@ type CoreDNS struct { type coreDNSConfig struct { Replicas int - ClusterDNSIP string + ClusterDNSIPs []string + ClusterDNSIPFamilies []string ClusterDomain string Image string PullPolicy string MaxUnavailableReplicas *uint DisablePodAntiAffinity bool DisablePodDisruptionBudget bool + DualStack bool } // NewCoreDNS creates new instance of CoreDNS component func NewCoreDNS(k0sVars *config.CfgVars, clientFactory k8sutil.ClientFactoryInterface, nodeConfig *v1beta1.ClusterConfig) (*CoreDNS, error) { - dnsAddress, err := nodeConfig.Spec.Network.DNSAddress(nodeConfig.Spec.PrimaryAddressFamily()) + dnsAddresses, err := nodeConfig.Spec.Network.DNSAddresses(nodeConfig.Spec.PrimaryAddressFamily()) if err != nil { return nil, err } @@ -310,7 +324,7 @@ func NewCoreDNS(k0sVars *config.CfgVars, clientFactory k8sutil.ClientFactoryInte } return &CoreDNS{ - dnsAddress: dnsAddress, + dnsAddresses: dnsAddresses, clusterDomain: nodeConfig.Spec.Network.ClusterDomain, client: client, log: logrus.WithField("component", "coredns"), @@ -361,12 +375,28 @@ func (c *CoreDNS) getConfig(ctx context.Context, clusterConfig *v1beta1.ClusterC nodeCount := len(nodes.Items) + var addresses []string + var addressFamilies []string + for _, address := range c.dnsAddresses { + addresses = append(addresses, address.String()) + if address == nil { + return coreDNSConfig{}, fmt.Errorf("invalid IP address %q", address) + } + if address.To4() == nil { + addressFamilies = append(addressFamilies, "IPv6") + } else { + addressFamilies = append(addressFamilies, "IPv4") + } + } + config := coreDNSConfig{ - Replicas: replicaCount(nodeCount), - ClusterDomain: c.clusterDomain, - ClusterDNSIP: c.dnsAddress, - Image: clusterConfig.Spec.Images.CoreDNS.URI(), - PullPolicy: clusterConfig.Spec.Images.DefaultPullPolicy, + Replicas: replicaCount(nodeCount), + ClusterDomain: c.clusterDomain, + ClusterDNSIPs: addresses, + ClusterDNSIPFamilies: addressFamilies, + Image: clusterConfig.Spec.Images.CoreDNS.URI(), + PullPolicy: clusterConfig.Spec.Images.DefaultPullPolicy, + DualStack: clusterConfig.Spec.Network.DualStack.Enabled, } if config.Replicas <= 1 { diff --git a/pkg/component/controller/coredns_test.go b/pkg/component/controller/coredns_test.go index 6372191456e9..51c12091fa43 100644 --- a/pkg/component/controller/coredns_test.go +++ b/pkg/component/controller/coredns_test.go @@ -18,7 +18,7 @@ func TestCoreDNS_RenderWithPatch(t *testing.T) { cfg := coreDNSConfig{ Replicas: 1, ClusterDomain: "cluster.local", - ClusterDNSIP: "10.96.0.10", + ClusterDNSIPs: []string{"10.96.0.10"}, Image: "coredns:latest", PullPolicy: "IfNotPresent", } diff --git a/pkg/component/controller/workerconfig/reconciler.go b/pkg/component/controller/workerconfig/reconciler.go index 5908c9c81563..7b3eb626f1f2 100644 --- a/pkg/component/controller/workerconfig/reconciler.go +++ b/pkg/component/controller/workerconfig/reconciler.go @@ -52,7 +52,7 @@ type Reconciler struct { log logrus.FieldLogger clusterDomain string - clusterDNSIP net.IP + clusterDNSIPs []net.IP clientFactory kubeutil.ClientFactoryInterface leaderElector leaderelector.Interface konnectivityEnabled bool @@ -89,20 +89,16 @@ var ( func NewReconciler(k0sVars *config.CfgVars, nodeConfig *v1beta1.ClusterConfig, clientFactory kubeutil.ClientFactoryInterface, leaderElector leaderelector.Interface, konnectivityEnabled, autopilotDisabled bool) (*Reconciler, error) { log := logrus.WithFields(logrus.Fields{"component": "workerconfig.Reconciler"}) - clusterDNSIPString, err := nodeConfig.Spec.Network.DNSAddress(nodeConfig.Spec.PrimaryAddressFamily()) + clusterDNSIPs, err := nodeConfig.Spec.Network.DNSAddresses(nodeConfig.Spec.PrimaryAddressFamily()) if err != nil { return nil, err } - clusterDNSIP := net.ParseIP(clusterDNSIPString) - if clusterDNSIP == nil { - return nil, fmt.Errorf("not an IP address: %q", clusterDNSIPString) - } reconciler := &Reconciler{ log: log, clusterDomain: nodeConfig.Spec.Network.ClusterDomain, - clusterDNSIP: clusterDNSIP, + clusterDNSIPs: clusterDNSIPs, clientFactory: clientFactory, leaderElector: leaderElector, konnectivityEnabled: konnectivityEnabled, @@ -579,6 +575,10 @@ func (r *Reconciler) buildProfile(snapshot *snapshot) *workerconfig.Profile { for i, cipherSuite := range constant.AllowedTLS12CipherSuiteIDs { cipherSuites[i] = tls.CipherSuiteName(cipherSuite) } + var clusterDNSIPs []string + for _, ip := range r.clusterDNSIPs { + clusterDNSIPs = append(clusterDNSIPs, ip.String()) + } workerProfile := &workerconfig.Profile{ APIServerAddresses: slices.Clone(snapshot.apiServers), @@ -588,7 +588,7 @@ func (r *Reconciler) buildProfile(snapshot *snapshot) *workerconfig.Profile { APIVersion: kubeletv1beta1.SchemeGroupVersion.String(), Kind: "KubeletConfiguration", }, - ClusterDNS: []string{r.clusterDNSIP.String()}, + ClusterDNS: clusterDNSIPs, ClusterDomain: r.clusterDomain, KubeReservedCgroup: "system.slice", KubeletCgroups: "/system.slice/containerd.service",