From 03479337da138b5a3f66753784e333bed7e1aee7 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Thu, 11 Jun 2026 14:03:48 -0400 Subject: [PATCH] feat: Allow configuring priorityClassName and terminationGracePeriodSeconds on the NLLB Envoy Pod Signed-off-by: Case Wylie --- docs/nllb.md | 23 ++++++++++++ pkg/apis/k0s/v1beta1/nllb.go | 27 +++++++++++++- pkg/apis/k0s/v1beta1/nllb_test.go | 35 +++++++++++++++++++ pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go | 5 +++ pkg/component/worker/nllb/envoy.go | 16 +++++++-- pkg/component/worker/nllb/envoy_test.go | 26 ++++++++++++++ .../k0s/k0s.k0sproject.io_clusterconfigs.yaml | 20 +++++++++++ 7 files changed, 148 insertions(+), 4 deletions(-) diff --git a/docs/nllb.md b/docs/nllb.md index 7f71e4ffafab..411eca1e5ba7 100644 --- a/docs/nllb.md +++ b/docs/nllb.md @@ -78,6 +78,29 @@ the new configuration to take effect. [specapi]: configuration.md#specapi +### Tuning the Envoy Pod + +The Envoy Pod that backs node-local load balancing is a static Pod managed by +k0s. A worker depends on it to reach the control plane, so it can be worthwhile +to make it more resilient and to control how it shuts down: + +```yaml +spec: + network: + nodeLocalLoadBalancing: + enabled: true + type: EnvoyProxy + envoyProxy: + # Protect the Envoy Pod from node-pressure eviction. The referenced + # PriorityClass must exist in the cluster. + priorityClassName: system-node-critical + # Give Envoy more time to drain in-flight connections on shutdown. + terminationGracePeriodSeconds: 60 +``` + +By default no priority class is assigned (the Pod's priority is zero) and +Kubernetes's default termination grace period of 30 seconds applies. + ## Full example using `k0sctl` The following example shows a full `k0sctl` configuration file featuring three diff --git a/pkg/apis/k0s/v1beta1/nllb.go b/pkg/apis/k0s/v1beta1/nllb.go index c747bf0af7a7..3436511fea2e 100644 --- a/pkg/apis/k0s/v1beta1/nllb.go +++ b/pkg/apis/k0s/v1beta1/nllb.go @@ -144,6 +144,23 @@ type EnvoyProxy struct { // +kubebuilder:validation:Minimum=1 // +kubebuilder:validation:Maximum=65535 KonnectivityServerBindPort *int32 `json:"konnectivityServerBindPort,omitempty"` + + // priorityClassName specifies the name of the PriorityClass that will be + // assigned to the Envoy Pod. As node-local load balancing is critical for a + // worker to reach the control plane, this can be used to protect the Envoy + // Pod from node-pressure eviction by assigning it a system-critical priority + // class, such as "system-node-critical". The referenced PriorityClass must + // exist, otherwise the Pod won't be admitted. If empty, no priority class is + // assigned and the Pod's priority defaults to zero. + PriorityClassName string `json:"priorityClassName,omitempty"` + + // terminationGracePeriodSeconds is the optional duration in seconds the + // Envoy Pod needs to terminate gracefully. If set, it overrides Kubernetes's + // default grace period of 30 seconds, e.g. to give Envoy more time to drain + // in-flight connections on shutdown. Must be a non-negative integer. A value + // of zero indicates that the Pod will be deleted immediately. + // +kubebuilder:validation:Minimum=0 + TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"` } // Describes configuration options required for using Traefik as the backing @@ -226,7 +243,15 @@ func (p *EnvoyProxy) Validate(path *field.Path) (errs field.ErrorList) { if p == nil { return } - return validateProxyConfig(path, p.Image, p.ImagePullPolicy, p.APIServerBindPort, p.KonnectivityServerBindPort) + errs = validateProxyConfig(path, p.Image, p.ImagePullPolicy, p.APIServerBindPort, p.KonnectivityServerBindPort) + if p.TerminationGracePeriodSeconds != nil && *p.TerminationGracePeriodSeconds < 0 { + errs = append(errs, field.Invalid( + path.Child("terminationGracePeriodSeconds"), + *p.TerminationGracePeriodSeconds, + "must be a non-negative integer", + )) + } + return errs } // Returns the default Traefik configuration. diff --git a/pkg/apis/k0s/v1beta1/nllb_test.go b/pkg/apis/k0s/v1beta1/nllb_test.go index a0eca98a5140..5a45ad0c54f1 100644 --- a/pkg/apis/k0s/v1beta1/nllb_test.go +++ b/pkg/apis/k0s/v1beta1/nllb_test.go @@ -7,6 +7,7 @@ import ( "fmt" "testing" + "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" "github.com/stretchr/testify/assert" @@ -82,6 +83,40 @@ spec: } } +func TestEnvoyProxy_PriorityClassAndGracePeriod(t *testing.T) { + yamlData := ` +apiVersion: k0s.k0sproject.io/v1beta1 +kind: ClusterConfig +metadata: + name: TestEnvoyProxy_PriorityClassAndGracePeriod +spec: + network: + nodeLocalLoadBalancing: + envoyProxy: + priorityClassName: system-node-critical + terminationGracePeriodSeconds: 60 +` + + t.Run("parsed", func(t *testing.T) { + c, err := ConfigFromBytes([]byte(yamlData)) + require.NoError(t, err) + require.Empty(t, c.Validate()) + envoy := c.Spec.Network.NodeLocalLoadBalancing.EnvoyProxy + require.NotNil(t, envoy) + assert.Equal(t, "system-node-critical", envoy.PriorityClassName) + require.NotNil(t, envoy.TerminationGracePeriodSeconds) + assert.Equal(t, int64(60), *envoy.TerminationGracePeriodSeconds) + }) + + t.Run("negative_grace_period_invalid", func(t *testing.T) { + envoy := &EnvoyProxy{TerminationGracePeriodSeconds: ptr.To(int64(-1))} + envoy.setDefaults() + errs := envoy.Validate(field.NewPath("envoyProxy")) + require.Len(t, errs, 1) + assert.ErrorContains(t, errs[0], "envoyProxy.terminationGracePeriodSeconds: Invalid value: -1: must be a non-negative integer") + }) +} + func TestEnvoyProxyImage_Unmarshal(t *testing.T) { yamlData := ` apiVersion: k0s.k0sproject.io/v1beta1 diff --git a/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go b/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go index ceae05821895..b56062904ab0 100644 --- a/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go @@ -610,6 +610,11 @@ func (in *EnvoyProxy) DeepCopyInto(out *EnvoyProxy) { *out = new(int32) **out = **in } + if in.TerminationGracePeriodSeconds != nil { + in, out := &in.TerminationGracePeriodSeconds, &out.TerminationGracePeriodSeconds + *out = new(int64) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvoyProxy. diff --git a/pkg/component/worker/nllb/envoy.go b/pkg/component/worker/nllb/envoy.go index 8e3d3d382bc2..c88993849740 100644 --- a/pkg/component/worker/nllb/envoy.go +++ b/pkg/component/worker/nllb/envoy.go @@ -65,6 +65,12 @@ type envoyPodParams struct { // The pull policy to use for the Envoy container. pullPolicy corev1.PullPolicy + + // The name of the PriorityClass to assign to the Envoy Pod. Empty if none. + priorityClassName string + + // The grace period for the Envoy Pod to terminate. Nil for the default. + terminationGracePeriodSeconds *int64 } // envoyFilesParams holds the parameters for the Envoy config files. @@ -139,8 +145,10 @@ func (e *envoyProxy) start(ctx context.Context, profile workerconfig.Profile, ap konnectivityBindPort, }, envoyPodParams{ - *nllb.EnvoyProxy.Image, - nllb.EnvoyProxy.ImagePullPolicy, + image: *nllb.EnvoyProxy.Image, + pullPolicy: nllb.EnvoyProxy.ImagePullPolicy, + priorityClassName: nllb.EnvoyProxy.PriorityClassName, + terminationGracePeriodSeconds: nllb.EnvoyProxy.TerminationGracePeriodSeconds, }, envoyFilesParams{ konnectivityServerPort: profile.Konnectivity.AgentPort, @@ -263,7 +271,9 @@ func makePodManifest(params *envoyParams, podParams *envoyPodParams) corev1.Pod Labels: applier.CommonLabels("nllb"), }, Spec: corev1.PodSpec{ - HostNetwork: true, + HostNetwork: true, + PriorityClassName: podParams.priorityClassName, + TerminationGracePeriodSeconds: podParams.terminationGracePeriodSeconds, SecurityContext: &corev1.PodSecurityContext{ RunAsNonRoot: ptr.To(true), }, diff --git a/pkg/component/worker/nllb/envoy_test.go b/pkg/component/worker/nllb/envoy_test.go index 97b6da6b4c4c..c5f55edb0508 100644 --- a/pkg/component/worker/nllb/envoy_test.go +++ b/pkg/component/worker/nllb/envoy_test.go @@ -12,14 +12,40 @@ import ( "testing" k0snet "github.com/k0sproject/k0s/internal/pkg/net" + "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" "k8s.io/client-go/util/jsonpath" + "k8s.io/utils/ptr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "sigs.k8s.io/yaml" ) +func TestMakePodManifest(t *testing.T) { + params := envoyParams{ + configDir: "/run/k0s/nllb/envoy", + bindIP: net.IPv6loopback, + } + + t.Run("defaults_leave_lifecycle_fields_unset", func(t *testing.T) { + pod := makePodManifest(¶ms, &envoyPodParams{image: v1beta1.ImageSpec{Image: "envoy", Version: "v1"}}) + assert.Empty(t, pod.Spec.PriorityClassName) + assert.Nil(t, pod.Spec.TerminationGracePeriodSeconds) + }) + + t.Run("propagates_priority_class_and_grace_period", func(t *testing.T) { + pod := makePodManifest(¶ms, &envoyPodParams{ + image: v1beta1.ImageSpec{Image: "envoy", Version: "v1"}, + priorityClassName: "system-node-critical", + terminationGracePeriodSeconds: ptr.To(int64(60)), + }) + assert.Equal(t, "system-node-critical", pod.Spec.PriorityClassName) + require.NotNil(t, pod.Spec.TerminationGracePeriodSeconds) + assert.Equal(t, int64(60), *pod.Spec.TerminationGracePeriodSeconds) + }) +} + func TestWriteEnvoyConfigFiles(t *testing.T) { for _, test := range []struct { name string diff --git a/static/_crds/k0s/k0s.k0sproject.io_clusterconfigs.yaml b/static/_crds/k0s/k0s.k0sproject.io_clusterconfigs.yaml index 82bd0809d779..765bb240b335 100644 --- a/static/_crds/k0s/k0s.k0sproject.io_clusterconfigs.yaml +++ b/static/_crds/k0s/k0s.k0sproject.io_clusterconfigs.yaml @@ -1271,6 +1271,26 @@ spec: maximum: 65535 minimum: 1 type: integer + priorityClassName: + description: |- + priorityClassName specifies the name of the PriorityClass that will be + assigned to the Envoy Pod. As node-local load balancing is critical for a + worker to reach the control plane, this can be used to protect the Envoy + Pod from node-pressure eviction by assigning it a system-critical priority + class, such as "system-node-critical". The referenced PriorityClass must + exist, otherwise the Pod won't be admitted. If empty, no priority class is + assigned and the Pod's priority defaults to zero. + type: string + terminationGracePeriodSeconds: + description: |- + terminationGracePeriodSeconds is the optional duration in seconds the + Envoy Pod needs to terminate gracefully. If set, it overrides Kubernetes's + default grace period of 30 seconds, e.g. to give Envoy more time to drain + in-flight connections on shutdown. Must be a non-negative integer. A value + of zero indicates that the Pod will be deleted immediately. + format: int64 + minimum: 0 + type: integer type: object traefik: description: |-