From 37b1cd3035c4e5a3df77f11da18249a2b9628d8a Mon Sep 17 00:00:00 2001 From: Joel Speed Date: Wed, 17 Dec 2025 12:18:58 +0000 Subject: [PATCH 1/7] Update library-go to include feature gating manifest-inclusion --- go.mod | 2 +- go.sum | 4 +- .../library-go/pkg/manifest/manifest.go | 65 ++++++++++++++++++- vendor/modules.txt | 2 +- 4 files changed, 66 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index e10c4e43af..2f6f0474de 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/openshift-eng/openshift-tests-extension v0.0.0-20250220212757-b9c4d98a0c45 github.com/openshift/api v0.0.0-20260116192047-6fb7fdae95fd github.com/openshift/client-go v0.0.0-20260108185524-48f4ccfc4e13 - github.com/openshift/library-go v0.0.0-20260108135436-db8dbd64c462 + github.com/openshift/library-go v0.0.0-20260121132910-dc3a1c884c04 github.com/operator-framework/api v0.17.1 github.com/operator-framework/operator-lifecycle-manager v0.22.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index efcc9ecd45..a129e4c4e4 100644 --- a/go.sum +++ b/go.sum @@ -90,8 +90,8 @@ github.com/openshift/api v0.0.0-20260116192047-6fb7fdae95fd h1:Hpjq/55Qb0Gy65Rew github.com/openshift/api v0.0.0-20260116192047-6fb7fdae95fd/go.mod h1:d5uzF0YN2nQQFA0jIEWzzOZ+edmo6wzlGLvx5Fhz4uY= github.com/openshift/client-go v0.0.0-20260108185524-48f4ccfc4e13 h1:6rd4zSo2UaWQcAPZfHK9yzKVqH0BnMv1hqMzqXZyTds= github.com/openshift/client-go v0.0.0-20260108185524-48f4ccfc4e13/go.mod h1:YvOmPmV7wcJxpfhTDuFqqs2Xpb3M3ovsM6Qs/i2ptq4= -github.com/openshift/library-go v0.0.0-20260108135436-db8dbd64c462 h1:zX9Od4Jg8sVmwQLwk6Vd+BX7tcyC/462FVvDdzHEPPk= -github.com/openshift/library-go v0.0.0-20260108135436-db8dbd64c462/go.mod h1:nIzWQQE49XbiKizVnVOip9CEB7HJ0hoJwNi3g3YKnKc= +github.com/openshift/library-go v0.0.0-20260121132910-dc3a1c884c04 h1:Fm9C8pT4l6VjpdqdhI1cBX9Y3D3S+rFxptVhCPBbMAI= +github.com/openshift/library-go v0.0.0-20260121132910-dc3a1c884c04/go.mod h1:nIzWQQE49XbiKizVnVOip9CEB7HJ0hoJwNi3g3YKnKc= github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12 h1:AKx/w1qpS8We43bsRgf8Nll3CGlDHpr/WAXvuedTNZI= github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/operator-framework/api v0.17.1 h1:J/6+Xj4IEV8C7hcirqUFwOiZAU3PbnJhWvB0/bB51c4= diff --git a/vendor/github.com/openshift/library-go/pkg/manifest/manifest.go b/vendor/github.com/openshift/library-go/pkg/manifest/manifest.go index d4082ce86b..80ccf12d85 100644 --- a/vendor/github.com/openshift/library-go/pkg/manifest/manifest.go +++ b/vendor/github.com/openshift/library-go/pkg/manifest/manifest.go @@ -26,6 +26,7 @@ const ( CapabilityAnnotation = "capability.openshift.io/name" DefaultClusterProfile = "self-managed-high-availability" featureSetAnnotation = "release.openshift.io/feature-set" + featureGateAnnotation = "release.openshift.io/feature-gate" ) var knownFeatureSets = sets.Set[string]{} @@ -171,6 +172,16 @@ func getFeatureSets(annotations map[string]string) (sets.Set[string], bool, erro return ret, specified, nil } +func hasFeatureSetAnnotation(annotations map[string]string) bool { + _, ok := annotations[featureSetAnnotation] + return ok +} + +func hasFeatureGateAnnotation(annotations map[string]string) bool { + _, ok := annotations[featureGateAnnotation] + return ok +} + func checkFeatureSets(requiredFeatureSet string, annotations map[string]string) error { requiredAnnotationValue := requiredFeatureSet if len(requiredFeatureSet) == 0 { @@ -187,12 +198,46 @@ func checkFeatureSets(requiredFeatureSet string, annotations map[string]string) return nil } +// checkFeatureGates validates if manifest should be included based on feature gate requirements +func checkFeatureGates(enabledGates sets.Set[string], annotations map[string]string) error { + if annotations == nil { + return nil // No annotations, include by default + } + gateRequirements, ok := annotations[featureGateAnnotation] + if !ok { + return nil // No requirements, include by default + } + + requirements := strings.Split(gateRequirements, ",") + for _, req := range requirements { + req = strings.TrimSpace(req) + if req == "" { + continue + } + + if strings.HasPrefix(req, "-") { + // Exclusion: gate must NOT be enabled + gate := req[1:] + if enabledGates.Has(gate) { + return fmt.Errorf("feature gate %s is enabled but manifest requires it to be disabled", gate) + } + } else { + // Inclusion: gate must be enabled + if !enabledGates.Has(req) { + return fmt.Errorf("feature gate %s is required but not enabled", req) + } + } + } + + return nil +} + // Include returns an error if the manifest fails an inclusion filter and should be excluded from further // processing by cluster version operator. Pointer arguments can be set nil to avoid excluding based on that // filter. For example, setting profile non-nil and capabilities nil will return an error if the manifest's // profile does not match, but will never return an error about capability issues. -func (m *Manifest) Include(excludeIdentifier *string, requiredFeatureSet *string, profile *string, capabilities *configv1.ClusterVersionCapabilitiesStatus, overrides []configv1.ComponentOverride) error { - return m.IncludeAllowUnknownCapabilities(excludeIdentifier, requiredFeatureSet, profile, capabilities, overrides, false) +func (m *Manifest) Include(excludeIdentifier *string, requiredFeatureSet *string, profile *string, capabilities *configv1.ClusterVersionCapabilitiesStatus, overrides []configv1.ComponentOverride, enabledFeatureGates sets.Set[string]) error { + return m.IncludeAllowUnknownCapabilities(excludeIdentifier, requiredFeatureSet, profile, capabilities, overrides, enabledFeatureGates, false) } // IncludeAllowUnknownCapabilities returns an error if the manifest fails an inclusion filter and should be excluded from @@ -202,7 +247,7 @@ func (m *Manifest) Include(excludeIdentifier *string, requiredFeatureSet *string // to capabilities filtering. When set to true a manifest will not be excluded simply because it contains an unknown // capability. This is necessary to allow updates to an OCP version containing newly defined capabilities. func (m *Manifest) IncludeAllowUnknownCapabilities(excludeIdentifier *string, requiredFeatureSet *string, profile *string, - capabilities *configv1.ClusterVersionCapabilitiesStatus, overrides []configv1.ComponentOverride, allowUnknownCapabilities bool) error { + capabilities *configv1.ClusterVersionCapabilitiesStatus, overrides []configv1.ComponentOverride, enabledFeatureGates sets.Set[string], allowUnknownCapabilities bool) error { annotations := m.Obj.GetAnnotations() if annotations == nil { @@ -216,6 +261,12 @@ func (m *Manifest) IncludeAllowUnknownCapabilities(excludeIdentifier *string, re } } + if requiredFeatureSet != nil && enabledFeatureGates != nil { + if hasFeatureSetAnnotation(annotations) && hasFeatureGateAnnotation(annotations) { + return fmt.Errorf("both feature set and feature gate annotations are present: manifests may specify either a feature set or a feature gate, but not both") + } + } + if requiredFeatureSet != nil { err := checkFeatureSets(*requiredFeatureSet, annotations) if err != nil { @@ -223,6 +274,14 @@ func (m *Manifest) IncludeAllowUnknownCapabilities(excludeIdentifier *string, re } } + // Feature gate filtering + if enabledFeatureGates != nil { + err := checkFeatureGates(enabledFeatureGates, annotations) + if err != nil { + return err + } + } + if profile != nil { profileAnnotation := fmt.Sprintf("include.release.openshift.io/%s", *profile) if val, ok := annotations[profileAnnotation]; ok && val != "true" { diff --git a/vendor/modules.txt b/vendor/modules.txt index 34c514e3dc..050891ed18 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -236,7 +236,7 @@ github.com/openshift/client-go/security/clientset/versioned/fake github.com/openshift/client-go/security/clientset/versioned/scheme github.com/openshift/client-go/security/clientset/versioned/typed/security/v1 github.com/openshift/client-go/security/clientset/versioned/typed/security/v1/fake -# github.com/openshift/library-go v0.0.0-20260108135436-db8dbd64c462 +# github.com/openshift/library-go v0.0.0-20260121132910-dc3a1c884c04 ## explicit; go 1.24.0 github.com/openshift/library-go/pkg/apiserver/jsonpatch github.com/openshift/library-go/pkg/config/clusterstatus From 226437ec6d4d287a17d13db159f9062a504274c3 Mon Sep 17 00:00:00 2001 From: Joel Speed Date: Wed, 17 Dec 2025 15:57:14 +0000 Subject: [PATCH 2/7] Pass feature gating information through to sync workers --- hack/cluster-version-util/task_graph.go | 3 +- lib/manifest/manifest.go | 5 + pkg/cvo/cvo.go | 95 +++++++- pkg/cvo/cvo_featuregates_test.go | 176 +++++++++++++++ pkg/cvo/cvo_scenarios_test.go | 64 ++++++ pkg/cvo/featuregate_integration_test.go | 280 ++++++++++++++++++++++++ pkg/cvo/status_test.go | 8 +- pkg/cvo/sync_test.go | 3 +- pkg/cvo/sync_worker.go | 90 +++++--- pkg/cvo/sync_worker_test.go | 21 +- pkg/featuregates/featuregates.go | 36 +++ pkg/payload/payload.go | 8 +- pkg/payload/payload_test.go | 6 +- pkg/payload/render.go | 2 +- pkg/payload/task_graph_test.go | 3 +- pkg/start/start.go | 22 +- pkg/start/start_integration_test.go | 12 +- 17 files changed, 766 insertions(+), 68 deletions(-) create mode 100644 pkg/cvo/cvo_featuregates_test.go create mode 100644 pkg/cvo/featuregate_integration_test.go diff --git a/hack/cluster-version-util/task_graph.go b/hack/cluster-version-util/task_graph.go index 122ccd7812..8f9c93baeb 100644 --- a/hack/cluster-version-util/task_graph.go +++ b/hack/cluster-version-util/task_graph.go @@ -6,6 +6,7 @@ import ( "time" "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/util/sets" "github.com/openshift/cluster-version-operator/pkg/payload" ) @@ -30,7 +31,7 @@ func newTaskGraphCmd() *cobra.Command { func runTaskGraphCmd(cmd *cobra.Command, args []string) error { manifestDir := args[0] - release, err := payload.LoadUpdate(manifestDir, "", "", "", payload.DefaultClusterProfile, nil) + release, err := payload.LoadUpdate(manifestDir, "", "", "", payload.DefaultClusterProfile, nil, sets.Set[string]{}) if err != nil { return err } diff --git a/lib/manifest/manifest.go b/lib/manifest/manifest.go index d5ad682331..9be5d76534 100644 --- a/lib/manifest/manifest.go +++ b/lib/manifest/manifest.go @@ -33,6 +33,9 @@ type InclusionConfiguration struct { // Platform, if non-nil, excludes CredentialsRequests manifests unless they match the infrastructure platform. Platform *string + + // EnabledFeatureGates excludes manifests with a feature gate requirement when the condition is not met. + EnabledFeatureGates sets.Set[string] } // GetImplicitlyEnabledCapabilities returns a set of capabilities that are implicitly enabled after a cluster update. @@ -57,6 +60,7 @@ func GetImplicitlyEnabledCapabilities( manifestInclusionConfiguration.Profile, manifestInclusionConfiguration.Capabilities, manifestInclusionConfiguration.Overrides, + manifestInclusionConfiguration.EnabledFeatureGates, true, ) // update manifest is enabled, no need to check @@ -74,6 +78,7 @@ func GetImplicitlyEnabledCapabilities( manifestInclusionConfiguration.Profile, manifestInclusionConfiguration.Capabilities, manifestInclusionConfiguration.Overrides, + manifestInclusionConfiguration.EnabledFeatureGates, true, ); err != nil { continue diff --git a/pkg/cvo/cvo.go b/pkg/cvo/cvo.go index 2def402752..2e5a1ea3fe 100644 --- a/pkg/cvo/cvo.go +++ b/pkg/cvo/cvo.go @@ -13,6 +13,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" informerscorev1 "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes" @@ -120,6 +121,7 @@ type Operator struct { cmConfigLister listerscorev1.ConfigMapNamespaceLister cmConfigManagedLister listerscorev1.ConfigMapNamespaceLister proxyLister configlistersv1.ProxyLister + featureGateLister configlistersv1.FeatureGateLister cacheSynced []cache.InformerSynced // queue tracks applying updates to a cluster. @@ -189,6 +191,10 @@ type Operator struct { // featurechangestopper controller will detect when cluster feature gate config changes and shutdown the CVO. enabledFeatureGates featuregates.CvoGateChecker + // featureGatesMutex protects access to enabledManifestFeatureGates + featureGatesMutex sync.RWMutex + enabledManifestFeatureGates sets.Set[string] + clusterProfile string uid types.UID @@ -213,6 +219,7 @@ func New( cmConfigManagedInformer informerscorev1.ConfigMapInformer, proxyInformer configinformersv1.ProxyInformer, operatorInformerFactory operatorexternalversions.SharedInformerFactory, + featureGateInformer configinformersv1.FeatureGateInformer, client clientset.Interface, kubeClient kubernetes.Interface, operatorClient operatorclientset.Interface, @@ -225,6 +232,7 @@ func New( alwaysEnableCapabilities []configv1.ClusterVersionCapability, featureSet configv1.FeatureSet, cvoGates featuregates.CvoGateChecker, + startingEnabledManifestFeatureGates sets.Set[string], ) (*Operator, error) { eventBroadcaster := record.NewBroadcaster() eventBroadcaster.StartLogging(klog.Infof) @@ -248,9 +256,9 @@ func New( kubeClient: kubeClient, operatorClient: operatorClient, eventRecorder: eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: namespace}), - queue: workqueue.NewTypedRateLimitingQueueWithConfig[any](workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "clusterversion"}), - availableUpdatesQueue: workqueue.NewTypedRateLimitingQueueWithConfig[any](workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "availableupdates"}), - upgradeableQueue: workqueue.NewTypedRateLimitingQueueWithConfig[any](workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "upgradeable"}), + queue: workqueue.NewTypedRateLimitingQueueWithConfig(workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "clusterversion"}), + availableUpdatesQueue: workqueue.NewTypedRateLimitingQueueWithConfig(workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "availableupdates"}), + upgradeableQueue: workqueue.NewTypedRateLimitingQueueWithConfig(workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "upgradeable"}), hypershift: hypershift, exclude: exclude, @@ -258,8 +266,9 @@ func New( conditionRegistry: standard.NewConditionRegistry(promqlTarget), injectClusterIdIntoPromQL: injectClusterIdIntoPromQL, - requiredFeatureSet: featureSet, - enabledFeatureGates: cvoGates, + requiredFeatureSet: featureSet, + enabledFeatureGates: cvoGates, + enabledManifestFeatureGates: startingEnabledManifestFeatureGates, alwaysEnableCapabilities: alwaysEnableCapabilities, } @@ -276,6 +285,9 @@ func New( if _, err := coInformer.Informer().AddEventHandler(optr.clusterOperatorEventHandler()); err != nil { return nil, err } + if _, err := featureGateInformer.Informer().AddEventHandler(optr.featureGateEventHandler()); err != nil { + return nil, err + } optr.coLister = coInformer.Lister() optr.cacheSynced = append(optr.cacheSynced, coInformer.Informer().HasSynced) @@ -287,6 +299,9 @@ func New( optr.cmConfigLister = cmConfigInformer.Lister().ConfigMaps(internal.ConfigNamespace) optr.cmConfigManagedLister = cmConfigManagedInformer.Lister().ConfigMaps(internal.ConfigManagedNamespace) + optr.featureGateLister = featureGateInformer.Lister() + optr.cacheSynced = append(optr.cacheSynced, featureGateInformer.Informer().HasSynced) + // make sure this is initialized after all the listers are initialized optr.upgradeableChecks = optr.defaultUpgradeableChecks() @@ -318,7 +333,7 @@ func (optr *Operator) LoadInitialPayload(ctx context.Context, restConfig *rest.C } update, err := payload.LoadUpdate(optr.defaultPayloadDir(), optr.release.Image, optr.exclude, string(optr.requiredFeatureSet), - optr.clusterProfile, configv1.KnownClusterVersionCapabilities) + optr.clusterProfile, configv1.KnownClusterVersionCapabilities, optr.getEnabledFeatureGates()) if err != nil { return nil, fmt.Errorf("the local release contents are invalid - no current version can be determined from disk: %v", err) @@ -779,7 +794,7 @@ func (optr *Operator) sync(ctx context.Context, key string) error { } // inform the config sync loop about our desired state - status := optr.configSync.Update(ctx, config.Generation, desired, config, state) + status := optr.configSync.Update(ctx, config.Generation, desired, config, state, optr.getEnabledFeatureGates()) // write cluster version status return optr.syncStatus(ctx, original, config, status, errs) @@ -1084,6 +1099,72 @@ func (optr *Operator) HTTPClient() (*http.Client, error) { }, nil } +// featureGateEventHandler handles changes to FeatureGate objects and updates the cluster feature gates +func (optr *Operator) featureGateEventHandler() cache.ResourceEventHandler { + workQueueKey := optr.queueKey() + return cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + if optr.updateEnabledFeatureGates(obj) { + optr.queue.Add(workQueueKey) + } + }, + UpdateFunc: func(old, new interface{}) { + if optr.updateEnabledFeatureGates(new) { + optr.queue.Add(workQueueKey) + } + }, + } +} + +// updateEnabledFeatureGates updates the cluster feature gates based on a FeatureGate object. +// Returns true or false based on whether or not the gates were actually updated. +// This allows us to avoid unnecessary work if the gates have not changed. +func (optr *Operator) updateEnabledFeatureGates(obj interface{}) bool { + featureGate, ok := obj.(*configv1.FeatureGate) + if !ok { + klog.Warningf("Expected FeatureGate object but got %T", obj) + return false + } + + newGates := optr.extractEnabledGates(featureGate) + + optr.featureGatesMutex.Lock() + defer optr.featureGatesMutex.Unlock() + + // Check if gates actually changed to avoid unnecessary work + if !optr.enabledManifestFeatureGates.Equal(newGates) { + + klog.V(2).Infof("Cluster feature gates changed from %v to %v", + sets.List(optr.enabledManifestFeatureGates), sets.List(newGates)) + + optr.enabledManifestFeatureGates = newGates + return true + } + + return false +} + +// getEnabledFeatureGates returns a copy of the current cluster feature gates for safe consumption +func (optr *Operator) getEnabledFeatureGates() sets.Set[string] { + optr.featureGatesMutex.RLock() + defer optr.featureGatesMutex.RUnlock() + + // Return a copy to prevent external modification + return optr.enabledManifestFeatureGates.Clone() +} + +// extractEnabledGates extracts the list of enabled feature gates for the current cluster version +func (optr *Operator) extractEnabledGates(featureGate *configv1.FeatureGate) sets.Set[string] { + // Find the feature gate details for the current loaded payload version. + currentVersion := optr.currentVersion().Version + if currentVersion == "" { + klog.Warningf("Payload has not been initialized yet, using the operator version %s", optr.enabledCVOFeatureGates.DesiredVersion()) + currentVersion = optr.enabledFeatureGates.DesiredVersion() + } + + return featuregates.ExtractEnabledGates(featureGate, currentVersion) +} + // shouldReconcileCVOConfiguration returns whether the CVO should reconcile its configuration using the API server. // // enabledFeatureGates must be initialized before the function is called. diff --git a/pkg/cvo/cvo_featuregates_test.go b/pkg/cvo/cvo_featuregates_test.go new file mode 100644 index 0000000000..d4d5e648fa --- /dev/null +++ b/pkg/cvo/cvo_featuregates_test.go @@ -0,0 +1,176 @@ +package cvo + +import ( + "testing" + + configv1 "github.com/openshift/api/config/v1" + "k8s.io/apimachinery/pkg/util/sets" +) + +func TestOperator_extractEnabledGates(t *testing.T) { + tests := []struct { + name string + featureGate *configv1.FeatureGate + release configv1.Release + expected sets.Set[string] + }{ + { + name: "extract gates for matching version", + featureGate: &configv1.FeatureGate{ + Status: configv1.FeatureGateStatus{ + FeatureGates: []configv1.FeatureGateDetails{ + { + Version: "4.14.0", + Enabled: []configv1.FeatureGateAttributes{ + {Name: "TechPreviewFeatureGate"}, + {Name: "ExperimentalFeature"}, + }, + }, + { + Version: "4.13.0", + Enabled: []configv1.FeatureGateAttributes{ + {Name: "OldFeature"}, + }, + }, + }, + }, + }, + release: configv1.Release{Version: "4.14.0"}, + expected: sets.New[string]("TechPreviewFeatureGate", "ExperimentalFeature"), + }, + { + name: "no matching version - return empty", + featureGate: &configv1.FeatureGate{ + Status: configv1.FeatureGateStatus{ + FeatureGates: []configv1.FeatureGateDetails{ + { + Version: "4.13.0", + Enabled: []configv1.FeatureGateAttributes{ + {Name: "OldFeature"}, + }, + }, + }, + }, + }, + release: configv1.Release{Version: "4.14.0"}, + expected: sets.Set[string]{}, + }, + { + name: "empty enabled gates", + featureGate: &configv1.FeatureGate{ + Status: configv1.FeatureGateStatus{ + FeatureGates: []configv1.FeatureGateDetails{ + { + Version: "4.14.0", + Enabled: []configv1.FeatureGateAttributes{}, + }, + }, + }, + }, + release: configv1.Release{Version: "4.14.0"}, + expected: sets.Set[string]{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + optr := &Operator{ + release: tt.release, + enabledFeatureGates: fakeRiFlags{ + desiredVersion: tt.release.Version, + }, + } + + result := optr.extractEnabledGates(tt.featureGate) + + if !result.Equal(tt.expected) { + t.Errorf("extractEnabledGates() = %v, expected %v", sets.List(result), sets.List(tt.expected)) + } + }) + } +} + +func TestOperator_getEnabledFeatureGates(t *testing.T) { + optr := &Operator{ + enabledManifestFeatureGates: sets.New[string]("gate1", "gate2"), + } + + result := optr.getEnabledFeatureGates() + expected := sets.New[string]("gate1", "gate2") + + if !result.Equal(expected) { + t.Errorf("getEnabledFeatureGates() = %v, expected %v", sets.List(result), sets.List(expected)) + } + + // Verify it returns a copy by modifying the result + result.Insert("gate3") + result2 := optr.getEnabledFeatureGates() + + if result2.Has("gate3") { + t.Error("getEnabledFeatureGates() should return a copy, but original was modified") + } +} + +func TestOperator_updateEnabledFeatureGates(t *testing.T) { + tests := []struct { + name string + obj interface{} + expectUpdate bool + }{ + { + name: "valid FeatureGate object", + obj: &configv1.FeatureGate{ + Status: configv1.FeatureGateStatus{ + FeatureGates: []configv1.FeatureGateDetails{ + { + Version: "4.14.0", + Enabled: []configv1.FeatureGateAttributes{ + {Name: "NewGate"}, + }, + }, + }, + }, + }, + expectUpdate: true, + }, + { + name: "invalid object type", + obj: "not-a-feature-gate", + expectUpdate: false, + }, + { + name: "nil object", + obj: nil, + expectUpdate: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + optr := &Operator{ + enabledManifestFeatureGates: sets.New[string]("oldgate"), + release: configv1.Release{Version: "4.14.0"}, + enabledFeatureGates: fakeRiFlags{ + desiredVersion: "4.14.0", + }, + } + + originalGates := optr.getEnabledFeatureGates() + optr.updateEnabledFeatureGates(tt.obj) + newGates := optr.getEnabledFeatureGates() + + if tt.expectUpdate { + if newGates.Equal(originalGates) { + t.Error("updateEnabledFeatureGates() expected gates to be updated") + } + if !newGates.Has("NewGate") { + t.Error("updateEnabledFeatureGates() expected NewGate to be enabled") + } + } else { + if !newGates.Equal(originalGates) { + t.Error("updateEnabledFeatureGates() should not update gates for invalid object") + } + } + }) + } +} diff --git a/pkg/cvo/cvo_scenarios_test.go b/pkg/cvo/cvo_scenarios_test.go index 8eeb8b8a42..11581c4f02 100644 --- a/pkg/cvo/cvo_scenarios_test.go +++ b/pkg/cvo/cvo_scenarios_test.go @@ -18,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" apiruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" dynamicfake "k8s.io/client-go/dynamic/fake" clientgotesting "k8s.io/client-go/testing" @@ -255,6 +256,7 @@ func TestCVO_StartupAndSync(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Generation: 1, @@ -266,6 +268,7 @@ func TestCVO_StartupAndSync(t *testing.T) { LastTransitionTime: time.Unix(2, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Generation: 1, @@ -291,6 +294,7 @@ func TestCVO_StartupAndSync(t *testing.T) { KnownCapabilities: sortedKnownCaps, }, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Generation: 1, @@ -317,6 +321,7 @@ func TestCVO_StartupAndSync(t *testing.T) { EnabledCapabilities: sortedCaps, }, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Generation: 1, @@ -343,6 +348,7 @@ func TestCVO_StartupAndSync(t *testing.T) { KnownCapabilities: sortedKnownCaps, }, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Generation: 1, @@ -370,6 +376,7 @@ func TestCVO_StartupAndSync(t *testing.T) { KnownCapabilities: sortedKnownCaps, }, }, + EnabledFeatureGates: sets.New[string](), }, ) @@ -452,6 +459,7 @@ func TestCVO_StartupAndSync(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Generation: 1, @@ -478,6 +486,7 @@ func TestCVO_StartupAndSync(t *testing.T) { LastTransitionTime: time.Unix(2, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Generation: 1, @@ -504,6 +513,7 @@ func TestCVO_StartupAndSync(t *testing.T) { LastTransitionTime: time.Unix(3, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Generation: 1, @@ -531,6 +541,7 @@ func TestCVO_StartupAndSync(t *testing.T) { LastTransitionTime: time.Unix(4, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, ) @@ -672,6 +683,7 @@ func TestCVO_StartupAndSyncUnverifiedPayload(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Actual: configv1.Release{ @@ -688,6 +700,7 @@ func TestCVO_StartupAndSyncUnverifiedPayload(t *testing.T) { Local: true, Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Total: 3, @@ -715,6 +728,7 @@ func TestCVO_StartupAndSyncUnverifiedPayload(t *testing.T) { Local: true, Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Done: 1, @@ -743,6 +757,7 @@ func TestCVO_StartupAndSyncUnverifiedPayload(t *testing.T) { Local: true, Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Initial: true, @@ -771,6 +786,7 @@ func TestCVO_StartupAndSyncUnverifiedPayload(t *testing.T) { Local: true, Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, ) @@ -860,6 +876,7 @@ func TestCVO_StartupAndSyncUnverifiedPayload(t *testing.T) { Local: true, Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Reconciling: true, @@ -888,6 +905,7 @@ func TestCVO_StartupAndSyncUnverifiedPayload(t *testing.T) { Local: true, Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Reconciling: true, @@ -916,6 +934,7 @@ func TestCVO_StartupAndSyncUnverifiedPayload(t *testing.T) { Local: true, Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Reconciling: true, @@ -945,6 +964,7 @@ func TestCVO_StartupAndSyncUnverifiedPayload(t *testing.T) { Local: true, Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, ) @@ -1076,6 +1096,7 @@ func TestCVO_StartupAndSyncPreconditionFailing(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Actual: configv1.Release{ @@ -1091,6 +1112,7 @@ func TestCVO_StartupAndSyncPreconditionFailing(t *testing.T) { Local: true, Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Total: 3, @@ -1117,6 +1139,7 @@ func TestCVO_StartupAndSyncPreconditionFailing(t *testing.T) { KnownCapabilities: sortedKnownCaps, }, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Done: 1, @@ -1144,6 +1167,7 @@ func TestCVO_StartupAndSyncPreconditionFailing(t *testing.T) { KnownCapabilities: sortedKnownCaps, }, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Done: 2, @@ -1171,6 +1195,7 @@ func TestCVO_StartupAndSyncPreconditionFailing(t *testing.T) { Local: true, Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, ) @@ -1258,6 +1283,7 @@ func TestCVO_StartupAndSyncPreconditionFailing(t *testing.T) { Local: true, Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Reconciling: true, @@ -1285,6 +1311,7 @@ func TestCVO_StartupAndSyncPreconditionFailing(t *testing.T) { Local: true, Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Reconciling: true, @@ -1312,6 +1339,7 @@ func TestCVO_StartupAndSyncPreconditionFailing(t *testing.T) { Local: true, Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Reconciling: true, @@ -1340,6 +1368,7 @@ func TestCVO_StartupAndSyncPreconditionFailing(t *testing.T) { Local: true, Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, ) @@ -1429,6 +1458,7 @@ func TestCVO_UpgradeUnverifiedPayload(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.1-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Actual: configv1.Release{Version: "1.0.1-abc", Image: "image/image:1"}, @@ -1441,6 +1471,7 @@ func TestCVO_UpgradeUnverifiedPayload(t *testing.T) { Update: configv1.Update{Version: "1.0.1-abc", Image: "image/image:1"}, Failure: payloadErr, }, + EnabledFeatureGates: sets.New[string](), }, ) actions = client.Actions() @@ -1557,6 +1588,7 @@ func TestCVO_UpgradeUnverifiedPayload(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.1-abc", Image: "image/image:1", Force: true}, }, + EnabledFeatureGates: sets.New[string](), }, ) @@ -1688,6 +1720,7 @@ func TestCVO_ResetPayloadLoadStatus(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.1-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Actual: configv1.Release{Version: "1.0.1-abc", Image: "image/image:1"}, @@ -1700,6 +1733,7 @@ func TestCVO_ResetPayloadLoadStatus(t *testing.T) { Update: configv1.Update{Version: "1.0.1-abc", Image: "image/image:1"}, Failure: payloadErr, }, + EnabledFeatureGates: sets.New[string](), }, ) actions = client.Actions() @@ -1817,6 +1851,7 @@ func TestCVO_ResetPayloadLoadStatus(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:0"}, }, + EnabledFeatureGates: sets.New[string](), }, ) actions = client.Actions() @@ -2095,6 +2130,7 @@ func TestCVO_InitImplicitlyEnabledCaps(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.1-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, ) actions := client.Actions() @@ -2222,6 +2258,7 @@ func TestCVO_UpgradeUnverifiedPayloadRetrieveOnce(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.1-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Actual: configv1.Release{Version: "1.0.1-abc", Image: "image/image:1"}, @@ -2234,6 +2271,7 @@ func TestCVO_UpgradeUnverifiedPayloadRetrieveOnce(t *testing.T) { Update: configv1.Update{Version: "1.0.1-abc", Image: "image/image:1"}, Failure: payloadErr, }, + EnabledFeatureGates: sets.New[string](), }, ) actions = client.Actions() @@ -2351,6 +2389,7 @@ func TestCVO_UpgradeUnverifiedPayloadRetrieveOnce(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.1-abc", Image: "image/image:1", Force: true}, }, + EnabledFeatureGates: sets.New[string](), }, ) @@ -2436,6 +2475,7 @@ func TestCVO_UpgradeUnverifiedPayloadRetrieveOnce(t *testing.T) { LastTransitionTime: time.Unix(4, 0), Update: configv1.Update{Version: "1.0.1-abc", Image: "image/image:1", Force: true}, }, + EnabledFeatureGates: sets.New[string](), }, finalStatusIndicatorCompleted, ) @@ -2510,6 +2550,7 @@ func TestCVO_UpgradePreconditionFailing(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.1-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Actual: configv1.Release{Version: "1.0.1-abc", Image: "image/image:1"}, @@ -2522,6 +2563,7 @@ func TestCVO_UpgradePreconditionFailing(t *testing.T) { Update: configv1.Update{Version: "1.0.1-abc", Image: "image/image:1"}, Failure: &payload.UpdateError{Reason: "UpgradePreconditionCheckFailed", Message: "Precondition \"TestPrecondition SuccessAfter: 3\" failed because of \"CheckFailure\": failing, attempt: 1 will succeed after 3 attempt", Name: "PreconditionCheck"}, }, + EnabledFeatureGates: sets.New[string](), }, ) @@ -2611,6 +2653,7 @@ func TestCVO_UpgradePreconditionFailing(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.1-abc", Image: "image/image:1", Force: true}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Done: 1, @@ -2635,6 +2678,7 @@ func TestCVO_UpgradePreconditionFailing(t *testing.T) { LastTransitionTime: time.Unix(2, 0), Update: configv1.Update{Version: "1.0.1-abc", Image: "image/image:1", Force: true}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Done: 2, @@ -2659,6 +2703,7 @@ func TestCVO_UpgradePreconditionFailing(t *testing.T) { LastTransitionTime: time.Unix(3, 0), Update: configv1.Update{Version: "1.0.1-abc", Image: "image/image:1", Force: true}, }, + EnabledFeatureGates: sets.New[string](), }, ) @@ -2785,6 +2830,7 @@ func TestCVO_UpgradePreconditionFailingAcceptedRisks(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.1-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Actual: configv1.Release{Version: "1.0.1-abc", Image: "image/image:1"}, @@ -2797,6 +2843,7 @@ func TestCVO_UpgradePreconditionFailingAcceptedRisks(t *testing.T) { Update: configv1.Update{Version: "1.0.1-abc", Image: "image/image:1"}, Failure: &payload.UpdateError{Reason: "UpgradePreconditionCheckFailed", Message: "Multiple precondition checks failed:\n* Precondition \"PreCondition1\" failed because of \"CheckFailure\": PreCondition1 will always fail.\n* Precondition \"PreCondition2\" failed because of \"CheckFailure\": PreCondition2 will always fail.", Name: "PreconditionCheck"}, }, + EnabledFeatureGates: sets.New[string](), }, ) @@ -2847,6 +2894,7 @@ func TestCVO_UpgradePreconditionFailingAcceptedRisks(t *testing.T) { LastTransitionTime: time.Unix(3, 0), Update: configv1.Update{Version: "1.0.1-abc", Image: "image/image:1", Force: true}, }, + EnabledFeatureGates: sets.New[string](), }, acceptedRisksPopulated, ) @@ -3151,6 +3199,7 @@ func TestCVO_RestartAndReconcile(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Reconciling: true, @@ -3165,6 +3214,7 @@ func TestCVO_RestartAndReconcile(t *testing.T) { LastTransitionTime: time.Unix(2, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Reconciling: true, @@ -3189,6 +3239,7 @@ func TestCVO_RestartAndReconcile(t *testing.T) { LastTransitionTime: time.Unix(3, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Reconciling: true, @@ -3214,6 +3265,7 @@ func TestCVO_RestartAndReconcile(t *testing.T) { LastTransitionTime: time.Unix(4, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Reconciling: true, @@ -3239,6 +3291,7 @@ func TestCVO_RestartAndReconcile(t *testing.T) { LastTransitionTime: time.Unix(5, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, ) client.ClearActions() @@ -3282,6 +3335,7 @@ func TestCVO_RestartAndReconcile(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Reconciling: true, @@ -3306,6 +3360,7 @@ func TestCVO_RestartAndReconcile(t *testing.T) { LastTransitionTime: time.Unix(2, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Reconciling: true, @@ -3331,6 +3386,7 @@ func TestCVO_RestartAndReconcile(t *testing.T) { LastTransitionTime: time.Unix(3, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Reconciling: true, @@ -3356,6 +3412,7 @@ func TestCVO_RestartAndReconcile(t *testing.T) { LastTransitionTime: time.Unix(4, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, ) client.ClearActions() @@ -3453,6 +3510,7 @@ func TestCVO_ErrorDuringReconcile(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, SyncWorkerStatus{ Reconciling: true, @@ -3467,6 +3525,7 @@ func TestCVO_ErrorDuringReconcile(t *testing.T) { LastTransitionTime: time.Unix(2, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, ) // verify we haven't observed any other events @@ -3515,6 +3574,7 @@ func TestCVO_ErrorDuringReconcile(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, ) clearAllStatus(t, worker.StatusCh()) @@ -3551,6 +3611,7 @@ func TestCVO_ErrorDuringReconcile(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, ) clearAllStatus(t, worker.StatusCh()) @@ -3602,6 +3663,7 @@ func TestCVO_ErrorDuringReconcile(t *testing.T) { LastTransitionTime: time.Unix(1, 0), Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }, ) client.ClearActions() @@ -3771,6 +3833,7 @@ func TestCVO_ParallelError(t *testing.T) { LastTransitionTime: status.loadPayloadStatus.LastTransitionTime, Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }) { t.Fatalf("unexpected status: %v", status) } @@ -3807,6 +3870,7 @@ func TestCVO_ParallelError(t *testing.T) { LastTransitionTime: status.loadPayloadStatus.LastTransitionTime, Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, }, + EnabledFeatureGates: sets.New[string](), }) { t.Fatalf("unexpected final: %v", status) } diff --git a/pkg/cvo/featuregate_integration_test.go b/pkg/cvo/featuregate_integration_test.go new file mode 100644 index 0000000000..f0545f4f10 --- /dev/null +++ b/pkg/cvo/featuregate_integration_test.go @@ -0,0 +1,280 @@ +package cvo + +import ( + "testing" + + configv1 "github.com/openshift/api/config/v1" + "github.com/openshift/library-go/pkg/manifest" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/sets" +) + +// TestFeatureGateManifestFiltering tests the end-to-end feature gate filtering pipeline +func TestFeatureGateManifestFiltering(t *testing.T) { + tests := []struct { + name string + enabledGates sets.Set[string] + manifestAnnotations map[string]string + shouldInclude bool + expectedError string + }{ + { + name: "include manifest with matching feature gate", + enabledGates: sets.New[string]("TechPreviewFeatureGate"), + manifestAnnotations: map[string]string{ + "release.openshift.io/feature-gate": "TechPreviewFeatureGate", + }, + shouldInclude: true, + }, + { + name: "exclude manifest with disabled feature gate", + enabledGates: sets.New[string]("SomeOtherGate"), + manifestAnnotations: map[string]string{ + "release.openshift.io/feature-gate": "TechPreviewFeatureGate", + }, + shouldInclude: false, + expectedError: "feature gate TechPreviewFeatureGate is required but not enabled", + }, + { + name: "include manifest when exclusion gate is disabled", + enabledGates: sets.New[string]("TechPreviewFeatureGate"), + manifestAnnotations: map[string]string{ + "release.openshift.io/feature-gate": "-DisabledFeature", + }, + shouldInclude: true, + }, + { + name: "exclude manifest when exclusion gate is enabled", + enabledGates: sets.New[string]("DisabledFeature"), + manifestAnnotations: map[string]string{ + "release.openshift.io/feature-gate": "-DisabledFeature", + }, + shouldInclude: false, + expectedError: "feature gate DisabledFeature is enabled but manifest requires it to be disabled", + }, + { + name: "complex filtering - AND logic", + enabledGates: sets.New[string]("FeatureA"), + manifestAnnotations: map[string]string{ + "release.openshift.io/feature-gate": "FeatureA,-FeatureB", + }, + shouldInclude: true, + }, + { + name: "complex filtering - failed AND logic", + enabledGates: sets.New[string]("FeatureA", "FeatureB"), + manifestAnnotations: map[string]string{ + "release.openshift.io/feature-gate": "FeatureA,-FeatureB", + }, + shouldInclude: false, + expectedError: "feature gate FeatureB is enabled but manifest requires it to be disabled", + }, + { + name: "manifest with no feature gate annotation", + enabledGates: sets.New[string]("AnyGate"), + manifestAnnotations: map[string]string{ + "some.other.annotation": "value", + }, + shouldInclude: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a mock manifest with the test annotations + obj := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "test-configmap", + "namespace": "test-namespace", + }, + }, + } + // Use SetAnnotations to ensure proper annotation handling + obj.SetAnnotations(tt.manifestAnnotations) + + manifest := &manifest.Manifest{ + Obj: obj, + } + + // Test the manifest inclusion logic + err := manifest.Include(nil, nil, nil, nil, nil, tt.enabledGates) + + if tt.shouldInclude { + if err != nil { + t.Errorf("Expected manifest to be included, but got error: %v", err) + } + } else { + if err == nil { + t.Error("Expected manifest to be excluded, but no error occurred") + } else if tt.expectedError != "" && err.Error() != tt.expectedError { + t.Errorf("Expected error %q, got %q", tt.expectedError, err.Error()) + } + } + }) + } +} + +// TestSyncWorkIntegration tests that feature gates are properly passed through the SyncWork pipeline +func TestSyncWorkIntegration(t *testing.T) { + work := &SyncWork{ + Generation: 1, + Desired: configv1.Update{Image: "test-image"}, + EnabledFeatureGates: sets.New[string]("TestGate1", "TestGate2"), + } + + // Test that the SyncWork can be used for manifest filtering + testObj := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "test-configmap", + }, + }, + } + testObj.SetAnnotations(map[string]string{ + "release.openshift.io/feature-gate": "TestGate1", + }) + + manifest := &manifest.Manifest{ + Obj: testObj, + } + + err := manifest.Include(nil, nil, nil, nil, nil, work.EnabledFeatureGates) + if err != nil { + t.Errorf("Expected manifest to be included with TestGate1 enabled, got error: %v", err) + } + + // Test with a gate that's not enabled + manifest.Obj.SetAnnotations(map[string]string{ + "release.openshift.io/feature-gate": "DisabledGate", + }) + + err = manifest.Include(nil, nil, nil, nil, nil, work.EnabledFeatureGates) + if err == nil { + t.Error("Expected manifest to be excluded with DisabledGate not enabled, but no error occurred") + } +} + +// TestFeatureGateEventHandling tests the feature gate event handler +func TestFeatureGateEventHandling(t *testing.T) { + // Create a simple operator with feature gate management capabilities + optr := &Operator{ + release: configv1.Release{Version: "4.14.0"}, + enabledFeatureGates: fakeRiFlags{ + desiredVersion: "4.14.0", + }, + } + + // Initialize feature gates to empty. + optr.updateEnabledFeatureGates(&configv1.FeatureGate{}) + + // Test that initial state is empty + gates := optr.getEnabledFeatureGates() + if gates.Len() != 0 { + t.Errorf("Expected empty initial feature gates, got %v", sets.List(gates)) + } + + // Test updating with a FeatureGate object + featureGate := &configv1.FeatureGate{ + Status: configv1.FeatureGateStatus{ + FeatureGates: []configv1.FeatureGateDetails{ + { + Version: "4.14.0", + Enabled: []configv1.FeatureGateAttributes{ + {Name: "NewFeature"}, + {Name: "ExperimentalFeature"}, + }, + }, + }, + }, + } + + optr.updateEnabledFeatureGates(featureGate) + + // Verify gates were updated + gates = optr.getEnabledFeatureGates() + expected := sets.New[string]("NewFeature", "ExperimentalFeature") + if !gates.Equal(expected) { + t.Errorf("After update, feature gates = %v, expected %v", sets.List(gates), sets.List(expected)) + } +} + +// TestManifestFilteringExamples tests real-world usage examples +func TestManifestFilteringExamples(t *testing.T) { + examples := []struct { + name string + description string + enabledGates sets.Set[string] + manifestAnnotation string + shouldInclude bool + }{ + { + name: "TechPreview feature deployment", + description: "Deploy experimental ConfigMap only in TechPreviewFeatureGate enabled clusters", + enabledGates: sets.New("TechPreviewFeatureGate"), + manifestAnnotation: "TechPreviewFeatureGate", + shouldInclude: true, + }, + { + name: "Production cluster excludes TechPreview", + description: "Production cluster should exclude TechPreviewFeatureGate enabled manifests", + enabledGates: sets.New[string](), + manifestAnnotation: "TechPreviewFeatureGate", + shouldInclude: false, + }, + { + name: "Alternative implementation selection", + description: "Use new storage API when enabled, exclude legacy API", + enabledGates: sets.New("NewStorageAPI"), + manifestAnnotation: "NewStorageAPI,-LegacyStorageAPI", + shouldInclude: true, + }, + { + name: "Legacy implementation when new API disabled", + description: "Use legacy implementation when new API is not enabled", + enabledGates: sets.New[string](), + manifestAnnotation: "-NewStorageAPI", + shouldInclude: true, + }, + } + + for _, example := range examples { + t.Run(example.name, func(t *testing.T) { + obj := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "example-manifest", + }, + }, + } + // Use SetAnnotations to ensure proper annotation handling + obj.SetAnnotations(map[string]string{ + "release.openshift.io/feature-gate": example.manifestAnnotation, + }) + + manifest := &manifest.Manifest{ + Obj: obj, + } + + err := manifest.Include(nil, nil, nil, nil, nil, example.enabledGates) + + if example.shouldInclude { + if err != nil { + t.Errorf("%s: Expected manifest to be included, but got error: %v", + example.description, err) + } + } else { + if err == nil { + t.Errorf("%s: Expected manifest to be excluded, but no error occurred", + example.description) + } + } + }) + } +} diff --git a/pkg/cvo/status_test.go b/pkg/cvo/status_test.go index bae07757db..f9c6f08216 100644 --- a/pkg/cvo/status_test.go +++ b/pkg/cvo/status_test.go @@ -3,11 +3,12 @@ package cvo import ( "context" "fmt" - "github.com/openshift/cluster-version-operator/pkg/internal" "reflect" "testing" "time" + "github.com/openshift/cluster-version-operator/pkg/internal" + "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/openshift/cluster-version-operator/pkg/payload" @@ -200,12 +201,17 @@ func TestOperator_syncFailingStatus(t *testing.T) { } type fakeRiFlags struct { + desiredVersion string unknownVersion bool statusReleaseArchitecture bool cvoConfiguration bool acceptRisks bool } +func (f fakeRiFlags) DesiredVersion() string { + return f.desiredVersion +} + func (f fakeRiFlags) UnknownVersion() bool { return f.unknownVersion } diff --git a/pkg/cvo/sync_test.go b/pkg/cvo/sync_test.go index a1dd8d9b01..a5bed45721 100644 --- a/pkg/cvo/sync_test.go +++ b/pkg/cvo/sync_test.go @@ -20,6 +20,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" dynamicfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/rest" clientgotesting "k8s.io/client-go/testing" @@ -493,7 +494,7 @@ func (r *fakeSyncRecorder) NotifyAboutManagedResourceActivity(message string) { func (r *fakeSyncRecorder) Start(ctx context.Context, maxWorkers int) { } -func (r *fakeSyncRecorder) Update(ctx context.Context, generation int64, desired configv1.Update, config *configv1.ClusterVersion, state payload.State) *SyncWorkerStatus { +func (r *fakeSyncRecorder) Update(ctx context.Context, generation int64, desired configv1.Update, config *configv1.ClusterVersion, state payload.State, enabledFeatureGates sets.Set[string]) *SyncWorkerStatus { r.Updates = append(r.Updates, desired) return r.Returns } diff --git a/pkg/cvo/sync_worker.go b/pkg/cvo/sync_worker.go index ebb72ef0a1..aa48d312fa 100644 --- a/pkg/cvo/sync_worker.go +++ b/pkg/cvo/sync_worker.go @@ -33,7 +33,7 @@ import ( // ConfigSyncWorker abstracts how the image is synchronized to the server. Introduced for testing. type ConfigSyncWorker interface { Start(ctx context.Context, maxWorkers int) - Update(ctx context.Context, generation int64, desired configv1.Update, config *configv1.ClusterVersion, state payload.State) *SyncWorkerStatus + Update(ctx context.Context, generation int64, desired configv1.Update, config *configv1.ClusterVersion, state payload.State, enabledFeatureGates sets.Set[string]) *SyncWorkerStatus StatusCh() <-chan SyncWorkerStatus // NotifyAboutManagedResourceActivity informs the sync worker about activity for a managed resource. @@ -78,6 +78,10 @@ type SyncWork struct { Attempt int Capabilities capability.ClusterCapabilities + + // EnabledFeatureGates contains the set of feature gate names that are currently enabled + // This is derived from the cluster's FeatureGate resource and used for manifest filtering + EnabledFeatureGates sets.Set[string] } // Empty returns true if the image is empty for this work. @@ -126,11 +130,20 @@ type SyncWorkerStatus struct { loadPayloadStatus LoadPayloadStatus CapabilitiesStatus CapabilityStatus + + // EnabledFeatureGates contains the set of feature gate names that are currently enabled + // and being used for manifest filtering during sync operations + EnabledFeatureGates sets.Set[string] } // DeepCopy copies the worker status. func (w SyncWorkerStatus) DeepCopy() *SyncWorkerStatus { - return &w + copy := w + + // Provide a proper deep copy for feature gates since this is a list. + copy.EnabledFeatureGates = w.EnabledFeatureGates.Clone() + + return © } // SyncWorker retrieves and applies the desired image, tracking the status for the parent to @@ -282,6 +295,18 @@ func (w *SyncWorker) syncPayload(ctx context.Context, work *SyncWork) ([]configv // cache the payload until the release image changes validPayload := w.payload + + previousEnabledFeatureGates := w.status.EnabledFeatureGates + if !work.EnabledFeatureGates.Equal(previousEnabledFeatureGates) { + // When the feature gates change, we must reload the payload. + // Loading the payload filters out files that didn't match the previous set of feature gates, + // this means now, additional files may match the new set of feature gates and need to be included. + // Some files in the current payload may no longer match the new set of feature gates and need to be excluded, + // though these ones are already excluded when apply calls Include on the manifests. + klog.V(2).Infof("Enabled feature gates changed from %v to %v, forcing a payload refresh", previousEnabledFeatureGates, work.EnabledFeatureGates) + w.payload = nil + } + if validPayload != nil && validPayload.Release.Image == desired.Image { // reset payload status to currently loaded payload if it no longer applies to desired target @@ -339,7 +364,7 @@ func (w *SyncWorker) syncPayload(ctx context.Context, work *SyncWork) ([]configv // Capability filtering is not done here since unknown capabilities are allowed // during updated payload load and enablement checking only occurs during apply. - payloadUpdate, err := payload.LoadUpdate(info.Directory, desired.Image, w.exclude, string(w.requiredFeatureSet), w.clusterProfile, nil) + payloadUpdate, err := payload.LoadUpdate(info.Directory, desired.Image, w.exclude, string(w.requiredFeatureSet), w.clusterProfile, nil, work.EnabledFeatureGates) if err != nil { msg := fmt.Sprintf("Loading payload failed version=%q image=%q failure=%v", desired.Version, desired.Image, err) @@ -415,7 +440,7 @@ func (w *SyncWorker) syncPayload(ctx context.Context, work *SyncWork) ([]configv } if w.payload != nil { implicitlyEnabledCaps = capability.SortedList(payload.GetImplicitlyEnabledCapabilities(payloadUpdate.Manifests, w.payload.Manifests, - work.Capabilities)) + work.Capabilities, work.EnabledFeatureGates)) } w.payload = payloadUpdate msg = fmt.Sprintf("Payload loaded version=%q image=%q architecture=%q", desired.Version, desired.Image, @@ -458,15 +483,16 @@ func (w *SyncWorker) loadUpdatedPayload(ctx context.Context, work *SyncWork) ([] // // Acquires the SyncWorker lock, so it must not be locked when Update is called func (w *SyncWorker) Update(ctx context.Context, generation int64, desired configv1.Update, config *configv1.ClusterVersion, - state payload.State) *SyncWorkerStatus { + state payload.State, enabledFeatureGates sets.Set[string]) *SyncWorkerStatus { w.lock.Lock() defer w.lock.Unlock() work := &SyncWork{ - Generation: generation, - Desired: desired, - Overrides: config.Spec.Overrides, + Generation: generation, + Desired: desired, + Overrides: config.Spec.Overrides, + EnabledFeatureGates: enabledFeatureGates, } var priorCaps sets.Set[configv1.ClusterVersionCapability] @@ -490,13 +516,13 @@ func (w *SyncWorker) Update(ctx context.Context, generation int64, desired confi ensureEnabledCapabilities := append(slices.Collect(maps.Keys(priorCaps)), w.alwaysEnableCapabilities...) work.Capabilities = capability.SetCapabilities(config, ensureEnabledCapabilities) - versionEqual, overridesEqual, capabilitiesEqual := + versionEqual, overridesEqual, capabilitiesEqual, featureGatesEqual := equalSyncWork(w.work, work, fmt.Sprintf("considering cluster version generation %d", generation)) // needs to be set here since changes in implicitly enabled capabilities are not considered a "capabilities change" w.status.CapabilitiesStatus.ImplicitlyEnabledCaps = capability.SortedList(work.Capabilities.ImplicitlyEnabled) - if versionEqual && overridesEqual && capabilitiesEqual { + if versionEqual && overridesEqual && capabilitiesEqual && featureGatesEqual { klog.V(2).Info("Update work is equal to current target; no change required") if !equalUpdate(w.work.Desired, w.status.loadPayloadStatus.Update) { @@ -520,6 +546,8 @@ func (w *SyncWorker) Update(ctx context.Context, generation int64, desired confi Version: work.Desired.Version, Image: work.Desired.Image, } + // Initialize feature gates in status + w.status.EnabledFeatureGates = work.EnabledFeatureGates.Clone() } else { oldDesired = &w.work.Desired } @@ -539,7 +567,9 @@ func (w *SyncWorker) Update(ctx context.Context, generation int64, desired confi if w.work != nil { w.work.Overrides = config.Spec.Overrides w.work.Capabilities = work.Capabilities + w.work.EnabledFeatureGates = work.EnabledFeatureGates.Clone() w.status.CapabilitiesStatus.Status = capability.GetCapabilitiesStatus(w.work.Capabilities) + w.status.EnabledFeatureGates = work.EnabledFeatureGates.Clone() } return w.status.DeepCopy() } @@ -557,7 +587,8 @@ func (w *SyncWorker) Update(ctx context.Context, generation int64, desired confi w.status.CapabilitiesStatus.ImplicitlyEnabledCaps = capability.SortedList(w.work.Capabilities.ImplicitlyEnabled) w.status.CapabilitiesStatus.Status = capability.GetCapabilitiesStatus(w.work.Capabilities) - // Update syncWorker status with architecture of newly loaded payload. + // Update syncWorker status with feature gates and architecture of newly loaded payload. + w.status.EnabledFeatureGates = w.work.EnabledFeatureGates.Clone() w.status.Architecture = w.payload.Architecture // notify the sync loop that we changed config @@ -763,8 +794,8 @@ func (w *statusWrapper) Report(status SyncWorkerStatus) { // time work transitions from empty to not empty (as a result of someone invoking // Update). func (w *SyncWork) calculateNextFrom(desired *SyncWork) bool { - sameVersion, sameOverrides, sameCapabilities := equalSyncWork(w, desired, "calculating next work") - changed := !sameVersion || !sameOverrides || !sameCapabilities + sameVersion, sameOverrides, sameCapabilities, sameFeatureGates := equalSyncWork(w, desired, "calculating next work") + changed := !sameVersion || !sameOverrides || !sameCapabilities || !sameFeatureGates // if this is the first time through the loop, initialize reconciling to // the state Update() calculated (to allow us to start in reconciling) @@ -784,6 +815,7 @@ func (w *SyncWork) calculateNextFrom(desired *SyncWork) bool { } w.Generation = desired.Generation + w.EnabledFeatureGates = desired.EnabledFeatureGates.Clone() return changed } @@ -820,20 +852,21 @@ func splitDigest(pullspec string) string { } // equalSyncWork returns indications of whether release version has changed, whether overrides have changed, -// and whether capabilities have changed. -func equalSyncWork(a, b *SyncWork, context string) (equalVersion, equalOverrides, equalCapabilities bool) { +// whether capabilities have changed, and whether enabled feature gates have changed. +func equalSyncWork(a, b *SyncWork, context string) (equalVersion, equalOverrides, equalCapabilities, equalFeatureGates bool) { // if both `a` and `b` are the same then simply return true if a == b { - return true, true, true + return true, true, true, true } // if either `a` or `b` are nil then return false if a == nil || b == nil { - return false, false, false + return false, false, false, false } sameVersion := equalUpdate(a.Desired, b.Desired) sameOverrides := reflect.DeepEqual(a.Overrides, b.Overrides) capabilitiesError := a.Capabilities.Equal(&b.Capabilities) + sameFeatureGates := a.EnabledFeatureGates.Equal(b.EnabledFeatureGates) var msgs []string if !sameVersion { @@ -845,10 +878,14 @@ func equalSyncWork(a, b *SyncWork, context string) (equalVersion, equalOverrides if capabilitiesError != nil { msgs = append(msgs, fmt.Sprintf("capabilities changed (%v)", capabilitiesError)) } + if !sameFeatureGates { + msgs = append(msgs, fmt.Sprintf("enabled feature gates changed (from %v to %v)", + sets.List(a.EnabledFeatureGates), sets.List(b.EnabledFeatureGates))) + } if len(msgs) > 0 { klog.V(2).Infof("Detected while %s: %s", context, strings.Join(msgs, ", ")) } - return sameVersion, sameOverrides, capabilitiesError == nil + return sameVersion, sameOverrides, capabilitiesError == nil, sameFeatureGates } // updateApplyStatus records the current status of the payload apply sync action for @@ -865,6 +902,7 @@ func (w *SyncWorker) updateApplyStatus(update SyncWorkerStatus) { // do not overwrite these status values which are not managed by apply update.loadPayloadStatus = w.status.loadPayloadStatus update.CapabilitiesStatus = w.status.CapabilitiesStatus + update.EnabledFeatureGates = w.status.EnabledFeatureGates.Clone() klog.V(6).Infof("Payload apply status change %#v", update) w.status = update @@ -1020,7 +1058,7 @@ func (w *SyncWorker) apply(ctx context.Context, work *SyncWork, maxWorkers int, if task.Manifest.GVK != configv1.GroupVersion.WithKind("ClusterOperator") { continue } - if err := task.Manifest.Include(nil, nil, nil, &capabilities, work.Overrides); err != nil { + if err := task.Manifest.Include(nil, nil, nil, &capabilities, work.Overrides, work.EnabledFeatureGates); err != nil { klog.V(manifestVerbosity).Infof("Skipping precreation of %s: %s", task, err) continue } @@ -1040,7 +1078,7 @@ func (w *SyncWorker) apply(ctx context.Context, work *SyncWork, maxWorkers int, klog.V(manifestVerbosity).Infof("Running sync for %s", task) - if err := task.Manifest.Include(nil, nil, nil, &capabilities, work.Overrides); err != nil { + if err := task.Manifest.Include(nil, nil, nil, &capabilities, work.Overrides, work.EnabledFeatureGates); err != nil { klog.V(manifestVerbosity).Infof("Skipping %s: %s", task, err) continue } @@ -1117,10 +1155,10 @@ func (r *consistentReporter) Update() { defer r.lock.Unlock() metricPayload.WithLabelValues(r.version, "pending").Set(float64(r.total - r.done)) metricPayload.WithLabelValues(r.version, "applied").Set(float64(r.done)) - copied := r.status + copied := r.status.DeepCopy() copied.Done = r.done copied.Total = r.total - r.reporter.Report(copied) + r.reporter.Report(*copied) } // Errors updates the status based on the current state of the graph runner. @@ -1131,13 +1169,13 @@ func (r *consistentReporter) Errors(errs []error) error { r.lock.Lock() defer r.lock.Unlock() - copied := r.status + copied := r.status.DeepCopy() copied.Done = r.done copied.Total = r.total if err != nil { copied.Failure = err } - r.reporter.Report(copied) + r.reporter.Report(*copied) return err } @@ -1155,13 +1193,13 @@ func (r *consistentReporter) Complete() { defer r.lock.Unlock() metricPayload.WithLabelValues(r.version, "pending").Set(float64(r.total - r.done)) metricPayload.WithLabelValues(r.version, "applied").Set(float64(r.done)) - copied := r.status + copied := r.status.DeepCopy() copied.Completed = r.completed + 1 copied.Initial = false copied.Reconciling = true copied.Done = r.done copied.Total = r.total - r.reporter.Report(copied) + r.reporter.Report(*copied) } func isContextError(err error) bool { diff --git a/pkg/cvo/sync_worker_test.go b/pkg/cvo/sync_worker_test.go index 3963f35450..4d62add4f9 100644 --- a/pkg/cvo/sync_worker_test.go +++ b/pkg/cvo/sync_worker_test.go @@ -15,6 +15,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" "github.com/openshift/cluster-version-operator/pkg/payload" @@ -31,55 +32,55 @@ func Test_statusWrapper_ReportProgress(t *testing.T) { { name: "skip updates that clear an error and are at an earlier fraction", previous: SyncWorkerStatus{Failure: fmt.Errorf("a"), Actual: configv1.Release{Image: "testing"}, Done: 10, Total: 100}, - next: SyncWorkerStatus{Actual: configv1.Release{Image: "testing"}}, + next: SyncWorkerStatus{Actual: configv1.Release{Image: "testing"}, EnabledFeatureGates: sets.New[string]()}, want: false, }, { previous: SyncWorkerStatus{Failure: fmt.Errorf("a"), Actual: configv1.Release{Image: "testing"}, Done: 10, Total: 100}, - next: SyncWorkerStatus{Actual: configv1.Release{Image: "testing2"}}, + next: SyncWorkerStatus{Actual: configv1.Release{Image: "testing2"}, EnabledFeatureGates: sets.New[string]()}, want: true, wantProgress: true, }, { previous: SyncWorkerStatus{Failure: fmt.Errorf("a"), Actual: configv1.Release{Image: "testing"}}, - next: SyncWorkerStatus{Actual: configv1.Release{Image: "testing"}}, + next: SyncWorkerStatus{Actual: configv1.Release{Image: "testing"}, EnabledFeatureGates: sets.New[string]()}, want: true, }, { previous: SyncWorkerStatus{Failure: fmt.Errorf("a"), Actual: configv1.Release{Image: "testing"}, Done: 10, Total: 100}, - next: SyncWorkerStatus{Failure: fmt.Errorf("a"), Actual: configv1.Release{Image: "testing"}}, + next: SyncWorkerStatus{Failure: fmt.Errorf("a"), Actual: configv1.Release{Image: "testing"}, EnabledFeatureGates: sets.New[string]()}, want: true, }, { previous: SyncWorkerStatus{Failure: fmt.Errorf("a"), Actual: configv1.Release{Image: "testing"}, Done: 10, Total: 100}, - next: SyncWorkerStatus{Failure: fmt.Errorf("b"), Actual: configv1.Release{Image: "testing"}, Done: 10, Total: 100}, + next: SyncWorkerStatus{Failure: fmt.Errorf("b"), Actual: configv1.Release{Image: "testing"}, Done: 10, Total: 100, EnabledFeatureGates: sets.New[string]()}, want: true, }, { previous: SyncWorkerStatus{Failure: fmt.Errorf("a"), Actual: configv1.Release{Image: "testing"}, Done: 10, Total: 100}, - next: SyncWorkerStatus{Failure: fmt.Errorf("b"), Actual: configv1.Release{Image: "testing"}, Done: 20, Total: 100}, + next: SyncWorkerStatus{Failure: fmt.Errorf("b"), Actual: configv1.Release{Image: "testing"}, Done: 20, Total: 100, EnabledFeatureGates: sets.New[string]()}, want: true, wantProgress: true, }, { previous: SyncWorkerStatus{Actual: configv1.Release{Image: "testing"}, Completed: 1}, - next: SyncWorkerStatus{Actual: configv1.Release{Image: "testing"}, Completed: 2}, + next: SyncWorkerStatus{Actual: configv1.Release{Image: "testing"}, Completed: 2, EnabledFeatureGates: sets.New[string]()}, want: true, wantProgress: true, }, { previous: SyncWorkerStatus{Actual: configv1.Release{Image: "testing-1"}, Completed: 1}, - next: SyncWorkerStatus{Actual: configv1.Release{Image: "testing-2"}, Completed: 1}, + next: SyncWorkerStatus{Actual: configv1.Release{Image: "testing-2"}, Completed: 1, EnabledFeatureGates: sets.New[string]()}, want: true, wantProgress: true, }, { previous: SyncWorkerStatus{Actual: configv1.Release{Image: "testing"}}, - next: SyncWorkerStatus{Actual: configv1.Release{Image: "testing"}}, + next: SyncWorkerStatus{Actual: configv1.Release{Image: "testing"}, EnabledFeatureGates: sets.New[string]()}, want: true, }, { - next: SyncWorkerStatus{Actual: configv1.Release{Image: "testing"}}, + next: SyncWorkerStatus{Actual: configv1.Release{Image: "testing"}, EnabledFeatureGates: sets.New[string]()}, want: true, wantProgress: true, }, diff --git a/pkg/featuregates/featuregates.go b/pkg/featuregates/featuregates.go index a66e19ca2b..093378f1ab 100644 --- a/pkg/featuregates/featuregates.go +++ b/pkg/featuregates/featuregates.go @@ -3,6 +3,8 @@ package featuregates import ( configv1 "github.com/openshift/api/config/v1" "github.com/openshift/api/features" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" ) // StubOpenShiftVersion is the default OpenShift version placeholder for the purpose of determining @@ -17,6 +19,10 @@ const StubOpenShiftVersion = "0.0.1-snapshot" // CvoGateChecker allows CVO code to check which feature gates are enabled type CvoGateChecker interface { + // DesiredVersion returns the version of the CVO that is currently executing. This is used to determine + // the feature gates that are relevant for the current version of the CVO. + DesiredVersion() string + // UnknownVersion flag is set to true if CVO did not find a matching version in the FeatureGate // status resource, meaning the current set of enabled and disabled feature gates is unknown for // this version. This should be a temporary state (config-operator should eventually add the @@ -55,6 +61,10 @@ type CvoGates struct { acceptRisks bool } +func (c CvoGates) DesiredVersion() string { + return c.desiredVersion +} + func (c CvoGates) StatusReleaseArchitecture() bool { return c.statusReleaseArchitecture } @@ -118,3 +128,29 @@ func CvoGatesFromFeatureGate(gate *configv1.FeatureGate, version string) CvoGate return enabledGates } + +// ExtractEnabledGates extracts the list of enabled feature gates for a given version from a FeatureGate object +// and returns a set of feature gate names. +// If no matching version is found, it returns an empty set. +func ExtractEnabledGates(featureGate *configv1.FeatureGate, currentVersion string) sets.Set[string] { + enabledGates := sets.Set[string]{} + + // Find the feature gate details for the current cluster version + for _, details := range featureGate.Status.FeatureGates { + if details.Version == currentVersion { + for _, enabled := range details.Enabled { + enabledGates.Insert(string(enabled.Name)) + } + klog.V(4).Infof("Found %d enabled feature gates for version %s: %v", + enabledGates.Len(), currentVersion, sets.List(enabledGates)) + break + } + } + + // If no matching version found, log a warning but continue with empty set + if enabledGates.Len() == 0 { + klog.V(2).Infof("No feature gates found for current version %s, using empty set", currentVersion) + } + + return enabledGates +} diff --git a/pkg/payload/payload.go b/pkg/payload/payload.go index 07d4abed60..1314cdca11 100644 --- a/pkg/payload/payload.go +++ b/pkg/payload/payload.go @@ -136,7 +136,7 @@ type metadata struct { } func LoadUpdate(dir, releaseImage, excludeIdentifier string, requiredFeatureSet string, profile string, - knownCapabilities []configv1.ClusterVersionCapability) (*Update, error) { + knownCapabilities []configv1.ClusterVersionCapability, enabledFeatureGates sets.Set[string]) (*Update, error) { klog.V(2).Infof("Loading updatepayload from %q", dir) if err := ValidateDirectory(dir); err != nil { return nil, err @@ -211,7 +211,7 @@ func LoadUpdate(dir, releaseImage, excludeIdentifier string, requiredFeatureSet filteredMs := []manifest.Manifest{} for _, manifest := range ms { - if err := manifest.Include(&excludeIdentifier, &requiredFeatureSet, &profile, onlyKnownCaps, nil); err != nil { + if err := manifest.Include(&excludeIdentifier, &requiredFeatureSet, &profile, onlyKnownCaps, nil, enabledFeatureGates); err != nil { klog.V(2).Infof("excluding %s: %v\n", manifest.String(), err) continue } @@ -243,12 +243,12 @@ func LoadUpdate(dir, releaseImage, excludeIdentifier string, requiredFeatureSet // the current payload the updated manifest's capabilities are checked to see if any must be implicitly enabled. // All capabilities requiring implicit enablement are returned. func GetImplicitlyEnabledCapabilities(updatePayloadManifests []manifest.Manifest, currentPayloadManifests []manifest.Manifest, - capabilities capability.ClusterCapabilities) sets.Set[configv1.ClusterVersionCapability] { + capabilities capability.ClusterCapabilities, enabledFeatureGates sets.Set[string]) sets.Set[configv1.ClusterVersionCapability] { clusterCaps := capability.GetCapabilitiesStatus(capabilities) return localmanifest.GetImplicitlyEnabledCapabilities( updatePayloadManifests, currentPayloadManifests, - localmanifest.InclusionConfiguration{Capabilities: &clusterCaps}, + localmanifest.InclusionConfiguration{Capabilities: &clusterCaps, EnabledFeatureGates: enabledFeatureGates}, capabilities.ImplicitlyEnabled, ) } diff --git a/pkg/payload/payload_test.go b/pkg/payload/payload_test.go index d828d02b72..21730fd23f 100644 --- a/pkg/payload/payload_test.go +++ b/pkg/payload/payload_test.go @@ -128,7 +128,7 @@ func TestLoadUpdate(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := LoadUpdate(tt.args.dir, tt.args.releaseImage, "exclude-test", "", DefaultClusterProfile, nil) + got, err := LoadUpdate(tt.args.dir, tt.args.releaseImage, "exclude-test", "", DefaultClusterProfile, nil, sets.Set[string]{}) if (err != nil) != tt.wantErr { t.Errorf("loadUpdatePayload() error = %v, wantErr %v", err, tt.wantErr) return @@ -203,7 +203,7 @@ func TestLoadUpdateArchitecture(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := LoadUpdate(tt.args.dir, tt.args.releaseImage, "exclude-test", "", DefaultClusterProfile, nil) + got, err := LoadUpdate(tt.args.dir, tt.args.releaseImage, "exclude-test", "", DefaultClusterProfile, nil, sets.Set[string]{}) if (err != nil) != tt.wantErr { t.Errorf("loadUpdatePayload() error = %v, wantErr %v", err, tt.wantErr) return @@ -365,7 +365,7 @@ func TestGetImplicitlyEnabledCapabilities(t *testing.T) { if tt.pathExt == "test10" { updateMans = append(updateMans, updateMans[0]) } - caps := GetImplicitlyEnabledCapabilities(updateMans, currentMans, tt.capabilities) + caps := GetImplicitlyEnabledCapabilities(updateMans, currentMans, tt.capabilities, sets.Set[string]{}) if diff := cmp.Diff(tt.wantImplicit, caps); diff != "" { t.Errorf("%s: wantImplicit differs from expected:\n%s", tt.name, diff) } diff --git a/pkg/payload/render.go b/pkg/payload/render.go index 03c6257ef1..5a00c2700d 100644 --- a/pkg/payload/render.go +++ b/pkg/payload/render.go @@ -166,7 +166,7 @@ func renderDir(renderConfig manifestRenderConfig, idir, odir string, requiredFea for _, manifest := range manifests { if !filterGroupKind.Has(manifest.GVK.GroupKind()) { klog.Infof("excluding %s because we do not render that group/kind", manifest.String()) - } else if err := manifest.Include(nil, requiredFeatureSet, clusterProfile, nil, nil); err != nil { + } else if err := manifest.Include(nil, requiredFeatureSet, clusterProfile, nil, nil, sets.Set[string]{} /* Empty, assuming any gated manifest will be applied later by the real CVO */); err != nil { klog.Infof("excluding %s: %v", manifest.String(), err) } else { filteredManifests = append(filteredManifests, string(manifest.Raw)) diff --git a/pkg/payload/task_graph_test.go b/pkg/payload/task_graph_test.go index 048bd93399..2e0d2208f9 100644 --- a/pkg/payload/task_graph_test.go +++ b/pkg/payload/task_graph_test.go @@ -16,6 +16,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/openshift/library-go/pkg/manifest" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" ) func Test_TaskGraph_Split(t *testing.T) { @@ -487,7 +488,7 @@ func Test_TaskGraph_real(t *testing.T) { if len(path) == 0 { t.Skip("TEST_GRAPH_PATH unset") } - p, err := LoadUpdate(path, "arbitrary/image:1", "", "", DefaultClusterProfile, nil) + p, err := LoadUpdate(path, "arbitrary/image:1", "", "", DefaultClusterProfile, nil, sets.Set[string]{}) if err != nil { t.Fatal(err) } diff --git a/pkg/start/start.go b/pkg/start/start.go index edbd9308e3..2a880d82cc 100644 --- a/pkg/start/start.go +++ b/pkg/start/start.go @@ -188,13 +188,13 @@ func (o *Options) Run(ctx context.Context) error { } clusterVersionConfigInformerFactory, configInformerFactory := o.prepareConfigInformerFactories(cb) - startingFeatureSet, startingCvoGates, err := o.processInitialFeatureGate(ctx, configInformerFactory) + startingFeatureSet, startingCvoGates, startingEnabledManifestFeatureGates, err := o.processInitialFeatureGate(ctx, configInformerFactory) if err != nil { return fmt.Errorf("error processing feature gates: %w", err) } // initialize the controllers and attempt to load the payload information - controllerCtx, err := o.NewControllerContext(cb, startingFeatureSet, startingCvoGates, clusterVersionConfigInformerFactory, configInformerFactory) + controllerCtx, err := o.NewControllerContext(cb, startingFeatureSet, startingCvoGates, startingEnabledManifestFeatureGates, clusterVersionConfigInformerFactory, configInformerFactory) if err != nil { return err } @@ -242,9 +242,10 @@ func (o *Options) getOpenShiftVersion() string { return releaseMetadata.Version } -func (o *Options) processInitialFeatureGate(ctx context.Context, configInformerFactory configinformers.SharedInformerFactory) (configv1.FeatureSet, featuregates.CvoGates, error) { +func (o *Options) processInitialFeatureGate(ctx context.Context, configInformerFactory configinformers.SharedInformerFactory) (configv1.FeatureSet, featuregates.CvoGates, sets.Set[string], error) { var startingFeatureSet configv1.FeatureSet var cvoGates featuregates.CvoGates + var startingEnabledManifestFeatureGates sets.Set[string] featureGates := configInformerFactory.Config().V1().FeatureGates().Lister() configInformerFactory.Start(ctx.Done()) @@ -254,7 +255,7 @@ func (o *Options) processInitialFeatureGate(ctx context.Context, configInformerF for key, synced := range configInformerFactory.WaitForCacheSync(ctx.Done()) { if !synced { - return startingFeatureSet, cvoGates, fmt.Errorf("failed to sync %s informer cache: %w", key.String(), ctx.Err()) + return startingFeatureSet, cvoGates, startingEnabledManifestFeatureGates, fmt.Errorf("failed to sync %s informer cache: %w", key.String(), ctx.Err()) } } @@ -268,16 +269,20 @@ func (o *Options) processInitialFeatureGate(ctx context.Context, configInformerF case apierrors.IsNotFound(err): // if we have no featuregates, then the cluster is using the default featureset, which is "". // This excludes everything that could possibly depend on a different feature set. + // Any manifest that blocks on a feature gate will be excluded from the cluster in this case. + // Since manifests roll-up to the default feature set over time, this should be safe and we will bring in + // the additional manifests once the feature gates become available. startingFeatureSet = "" - klog.Infof("FeatureGate not found in cluster, will assume default feature set %q at startup", startingFeatureSet) + klog.Infof("FeatureGate not found in cluster, will assume default feature set %q at startup, all feature gates will be disabled", startingFeatureSet) case err != nil: // This should not happen because featureGates is backed by the informer cache which successfully synced earlier klog.Errorf("Failed to get FeatureGate from cluster: %v", err) - return startingFeatureSet, cvoGates, fmt.Errorf("failed to get FeatureGate from informer cache: %w", err) + return startingFeatureSet, cvoGates, startingEnabledManifestFeatureGates, fmt.Errorf("failed to get FeatureGate from informer cache: %w", err) default: clusterFeatureGate = gate startingFeatureSet = gate.Spec.FeatureSet cvoGates = featuregates.CvoGatesFromFeatureGate(clusterFeatureGate, cvoOpenShiftVersion) + startingEnabledManifestFeatureGates = featuregates.ExtractEnabledGates(clusterFeatureGate, cvoOpenShiftVersion) klog.Infof("FeatureGate found in cluster, using its feature set %q at startup", startingFeatureSet) } @@ -286,7 +291,7 @@ func (o *Options) processInitialFeatureGate(ctx context.Context, configInformerF } klog.Infof("CVO features for version %s enabled at startup: %+v", cvoOpenShiftVersion, cvoGates) - return startingFeatureSet, cvoGates, nil + return startingFeatureSet, cvoGates, startingEnabledManifestFeatureGates, nil } // run launches a number of goroutines to handle manifest application, @@ -578,6 +583,7 @@ func (o *Options) NewControllerContext( cb *ClientBuilder, startingFeatureSet configv1.FeatureSet, startingCvoGates featuregates.CvoGates, + startingEnabledManifestFeatureGates sets.Set[string], clusterVersionConfigInformerFactory, configInformerFactory configinformers.SharedInformerFactory, ) (*Context, error) { @@ -608,6 +614,7 @@ func (o *Options) NewControllerContext( openshiftConfigManagedInformerFactory.Core().V1().ConfigMaps(), configInformerFactory.Config().V1().Proxies(), operatorInformerFactory, + configInformerFactory.Config().V1().FeatureGates(), cb.ClientOrDie(o.Namespace), cvoKubeClient, operatorClient, @@ -620,6 +627,7 @@ func (o *Options) NewControllerContext( stringsToCapabilities(o.AlwaysEnableCapabilities), startingFeatureSet, startingCvoGates, + startingEnabledManifestFeatureGates, ) if err != nil { return nil, err diff --git a/pkg/start/start_integration_test.go b/pkg/start/start_integration_test.go index 2322e215e6..22b40d4bf7 100644 --- a/pkg/start/start_integration_test.go +++ b/pkg/start/start_integration_test.go @@ -196,11 +196,11 @@ func TestIntegrationCVO_initializeAndUpgrade(t *testing.T) { } clusterVersionConfigInformerFactory, configInformerFactory := options.prepareConfigInformerFactories(cb) - featureset, gates, err := options.processInitialFeatureGate(context.Background(), configInformerFactory) + featureset, cvoGates, startingEnabledManifestFeatureGates, err := options.processInitialFeatureGate(context.Background(), configInformerFactory) if err != nil { t.Fatal(err) } - controllers, err := options.NewControllerContext(cb, featureset, gates, clusterVersionConfigInformerFactory, configInformerFactory) + controllers, err := options.NewControllerContext(cb, featureset, cvoGates, startingEnabledManifestFeatureGates, clusterVersionConfigInformerFactory, configInformerFactory) if err != nil { t.Fatal(err) } @@ -341,11 +341,11 @@ func TestIntegrationCVO_gracefulStepDown(t *testing.T) { } clusterVersionConfigInformerFactory, configInformerFactory := options.prepareConfigInformerFactories(cb) - featureset, gates, err := options.processInitialFeatureGate(context.Background(), configInformerFactory) + featureset, cvoGates, startingEnabledManifestFeatureGates, err := options.processInitialFeatureGate(context.Background(), configInformerFactory) if err != nil { t.Fatal(err) } - controllers, err := options.NewControllerContext(cb, featureset, gates, clusterVersionConfigInformerFactory, configInformerFactory) + controllers, err := options.NewControllerContext(cb, featureset, cvoGates, startingEnabledManifestFeatureGates, clusterVersionConfigInformerFactory, configInformerFactory) if err != nil { t.Fatal(err) } @@ -548,11 +548,11 @@ metadata: } clusterVersionConfigInformerFactory, configInformerFactory := options.prepareConfigInformerFactories(cb) - featureset, gates, err := options.processInitialFeatureGate(context.Background(), configInformerFactory) + featureset, cvoGates, startingEnabledManifestFeatureGates, err := options.processInitialFeatureGate(context.Background(), configInformerFactory) if err != nil { t.Fatal(err) } - controllers, err := options.NewControllerContext(cb, featureset, gates, clusterVersionConfigInformerFactory, configInformerFactory) + controllers, err := options.NewControllerContext(cb, featureset, cvoGates, startingEnabledManifestFeatureGates, clusterVersionConfigInformerFactory, configInformerFactory) if err != nil { t.Fatal(err) } From 61c52fb64e4c9a4d33cefbc97949345d92720fea Mon Sep 17 00:00:00 2001 From: Joel Speed Date: Fri, 19 Dec 2025 13:24:52 +0000 Subject: [PATCH 3/7] Filter all manifests at render based on featureset, featuregates and cluster profile --- pkg/payload/render.go | 148 ++++++++++++++++++++++++------------------ 1 file changed, 84 insertions(+), 64 deletions(-) diff --git a/pkg/payload/render.go b/pkg/payload/render.go index 5a00c2700d..e1c22e625d 100644 --- a/pkg/payload/render.go +++ b/pkg/payload/render.go @@ -12,7 +12,7 @@ import ( configv1 "github.com/openshift/api/config/v1" "github.com/openshift/library-go/pkg/manifest" "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" @@ -35,33 +35,9 @@ func Render(outputDir, releaseImage, featureGateManifestPath, clusterProfile str } ) - var requiredFeatureSet *string - if featureGateManifestPath != "" { - manifests, err := manifest.ManifestsFromFiles([]string{featureGateManifestPath}) - if err != nil { - return fmt.Errorf("loading FeatureGate manifest: %w", err) - } - if len(manifests) != 1 { - return fmt.Errorf("FeatureGate manifest %s contains %d manifests, but expected only one", featureGateManifestPath, len(manifests)) - } - featureGateManifest := manifests[0] - expectedGVK := schema.GroupVersionKind{Kind: "FeatureGate", Version: configv1.GroupVersion.Version, Group: config.GroupName} - if featureGateManifest.GVK != expectedGVK { - return fmt.Errorf("FeatureGate manifest %s GroupVersionKind %v, but expected %v", featureGateManifest.OriginalFilename, featureGateManifest.GVK, expectedGVK) - } - featureSet, found, err := unstructured.NestedString(featureGateManifest.Obj.Object, "spec", "featureSet") - if err != nil { - return fmt.Errorf("%s spec.featureSet lookup was not a string: %w", featureGateManifest.String(), err) - } else if found { - requiredFeatureSet = &featureSet - klog.Infof("--feature-gate-manifest-path=%s results in feature set %q", featureGateManifest.OriginalFilename, featureSet) - } else { - requiredFeatureSet = ptr.To[string]("") - klog.Infof("--feature-gate-manifest-path=%s does not set spec.featureSet, using the default feature set", featureGateManifest.OriginalFilename) - } - } else { - requiredFeatureSet = ptr.To[string]("") - klog.Info("--feature-gate-manifest-path is unset, using the default feature set") + requiredFeatureSet, enabledFeatureGates, err := parseFeatureGateManifest(featureGateManifestPath) + if err != nil { + return fmt.Errorf("error parsing feature gate manifest: %w", err) } tasks := []struct { @@ -94,7 +70,7 @@ func Render(outputDir, releaseImage, featureGateManifestPath, clusterProfile str }} var errs []error for _, task := range tasks { - if err := renderDir(renderConfig, task.idir, task.odir, requiredFeatureSet, &clusterProfile, task.processTemplate, task.skipFiles, task.filterGroupKind); err != nil { + if err := renderDir(renderConfig, task.idir, task.odir, requiredFeatureSet, enabledFeatureGates, &clusterProfile, task.processTemplate, task.skipFiles, task.filterGroupKind); err != nil { errs = append(errs, err) } } @@ -106,7 +82,9 @@ func Render(outputDir, releaseImage, featureGateManifestPath, clusterProfile str return nil } -func renderDir(renderConfig manifestRenderConfig, idir, odir string, requiredFeatureSet *string, clusterProfile *string, processTemplate bool, skipFiles sets.Set[string], filterGroupKind sets.Set[schema.GroupKind]) error { +func renderDir(renderConfig manifestRenderConfig, idir, odir string, requiredFeatureSet *string, enabledFeatureGates sets.Set[string], clusterProfile *string, processTemplate bool, skipFiles sets.Set[string], filterGroupKind sets.Set[schema.GroupKind]) error { + klog.Infof("Filtering manifests in %s for feature set %v, cluster profile %v and enabled feature gates %v", idir, *requiredFeatureSet, *clusterProfile, enabledFeatureGates.UnsortedList()) + if err := os.MkdirAll(odir, 0666); err != nil { return err } @@ -126,16 +104,6 @@ func renderDir(renderConfig manifestRenderConfig, idir, odir string, requiredFea if skipFiles.Has(file.Name()) { continue } - if strings.Contains(file.Name(), "CustomNoUpgrade") || - strings.Contains(file.Name(), "TechPreviewNoUpgrade") || - strings.Contains(file.Name(), "DevPreviewNoUpgrade") || - strings.Contains(file.Name(), "OKD") { - // CustomNoUpgrade, TechPreviewNoUpgrade, DevPreviewNoUpgrade, and OKD may add features to manifests like the ClusterVersion CRD, - // but we do not need those features during bootstrap-render time. In those clusters, the production - // CVO will be along shortly to update the manifests and deliver the gated features. - // fixme: now that we have requiredFeatureSet, use it to do Manifest.Include() filtering here instead of making filename assumptions - continue - } ipath := filepath.Join(idir, file.Name()) iraw, err := os.ReadFile(ipath) @@ -155,34 +123,32 @@ func renderDir(renderConfig manifestRenderConfig, idir, odir string, requiredFea rraw = iraw } - if len(filterGroupKind) > 0 { - manifests, err := manifest.ParseManifests(bytes.NewReader(rraw)) - if err != nil { - errs = append(errs, fmt.Errorf("parse manifest %s from %s: %w", file.Name(), idir, err)) - continue - } + manifests, err := manifest.ParseManifests(bytes.NewReader(rraw)) + if err != nil { + errs = append(errs, fmt.Errorf("parse manifest %s from %s: %w", file.Name(), idir, err)) + continue + } - filteredManifests := make([]string, 0, len(manifests)) - for _, manifest := range manifests { - if !filterGroupKind.Has(manifest.GVK.GroupKind()) { - klog.Infof("excluding %s because we do not render that group/kind", manifest.String()) - } else if err := manifest.Include(nil, requiredFeatureSet, clusterProfile, nil, nil, sets.Set[string]{} /* Empty, assuming any gated manifest will be applied later by the real CVO */); err != nil { - klog.Infof("excluding %s: %v", manifest.String(), err) - } else { - filteredManifests = append(filteredManifests, string(manifest.Raw)) - klog.Infof("including %s filtered by feature set %v and cluster profile %v", manifest.String(), requiredFeatureSet, clusterProfile) - } + filteredManifests := make([]string, 0, len(manifests)) + for _, manifest := range manifests { + if len(filterGroupKind) > 0 && !filterGroupKind.Has(manifest.GVK.GroupKind()) { + klog.Infof("excluding %s because we do not render that group/kind", manifest.String()) + } else if err := manifest.Include(nil, requiredFeatureSet, clusterProfile, nil, nil, enabledFeatureGates); err != nil { + klog.Infof("excluding %s: %v", manifest.String(), err) + } else { + filteredManifests = append(filteredManifests, string(manifest.Raw)) + klog.Infof("including %s", manifest.String()) } + } - if len(filteredManifests) == 0 { - continue - } + if len(filteredManifests) == 0 { + continue + } - if len(filteredManifests) == 1 { - rraw = []byte(filteredManifests[0]) - } else { - rraw = []byte(strings.Join(filteredManifests, "\n---\n")) - } + if len(filteredManifests) == 1 { + rraw = []byte(filteredManifests[0]) + } else { + rraw = []byte(strings.Join(filteredManifests, "\n---\n")) } opath := filepath.Join(odir, file.Name()) @@ -218,3 +184,57 @@ func renderManifest(config manifestRenderConfig, manifestBytes []byte) ([]byte, return buf.Bytes(), nil } + +func parseFeatureGateManifest(featureGateManifestPath string) (*string, sets.Set[string], error) { + if featureGateManifestPath == "" { + return ptr.To(""), sets.Set[string]{}, nil + } + + manifests, err := manifest.ManifestsFromFiles([]string{featureGateManifestPath}) + if err != nil { + return nil, nil, fmt.Errorf("loading FeatureGate manifest: %w", err) + } + + if len(manifests) != 1 { + return nil, nil, fmt.Errorf("FeatureGate manifest %s contains %d manifests, but expected only one", featureGateManifestPath, len(manifests)) + } + + featureGateManifest := manifests[0] + expectedGVK := schema.GroupVersionKind{Kind: "FeatureGate", Version: configv1.GroupVersion.Version, Group: config.GroupName} + if featureGateManifest.GVK != expectedGVK { + return nil, nil, fmt.Errorf("FeatureGate manifest %s GroupVersionKind %v, but expected %v", featureGateManifest.OriginalFilename, featureGateManifest.GVK, expectedGVK) + } + + // Convert unstructured object to structured FeatureGate + var featureGate configv1.FeatureGate + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(featureGateManifest.Obj.Object, &featureGate); err != nil { + return nil, nil, fmt.Errorf("failed to convert FeatureGate manifest %s to structured object: %w", featureGateManifest.OriginalFilename, err) + } + + var requiredFeatureSet *string + if featureGate.Spec.FeatureSet != "" { + featureSetString := string(featureGate.Spec.FeatureSet) + requiredFeatureSet = &featureSetString + klog.Infof("--feature-gate-manifest-path=%s results in feature set %q", featureGateManifest.OriginalFilename, featureGate.Spec.FeatureSet) + } else { + requiredFeatureSet = ptr.To("") + klog.Infof("--feature-gate-manifest-path=%s does not set spec.featureSet, using the default feature set", featureGateManifest.OriginalFilename) + } + + if len(featureGate.Status.FeatureGates) == 0 { + // In case there are no feature gates, fall back to feature set only behaviour. + return requiredFeatureSet, sets.Set[string]{}, nil + } + + // A rendered manifest should only include 1 version of the enabled feature gates. + if len(featureGate.Status.FeatureGates) > 1 { + return nil, nil, fmt.Errorf("FeatureGate manifest %s contains %d feature gates, but expected exactly one", featureGateManifest.OriginalFilename, len(featureGate.Status.FeatureGates)) + } + + enabledFeatureGates := sets.Set[string]{} + for _, feature := range featureGate.Status.FeatureGates[0].Enabled { + enabledFeatureGates.Insert(string(feature.Name)) + } + + return requiredFeatureSet, enabledFeatureGates, nil +} From 10b7a6c528a4acbbcddc34717e444da38383aecd Mon Sep 17 00:00:00 2001 From: Joel Speed Date: Fri, 19 Dec 2025 14:54:38 +0000 Subject: [PATCH 4/7] Ensure bootstrap pod is included in render By implementing full manifest filtering in the render step, we now need to make sure any manifest that we render includes a cluster profile. In this case, we just render it out with the cluster profile that the cluster is on. --- bootstrap/bootstrap-pod.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bootstrap/bootstrap-pod.yaml b/bootstrap/bootstrap-pod.yaml index 1a885ea801..580eebd49b 100644 --- a/bootstrap/bootstrap-pod.yaml +++ b/bootstrap/bootstrap-pod.yaml @@ -5,6 +5,8 @@ metadata: namespace: openshift-cluster-version labels: k8s-app: cluster-version-operator + annotations: + include.release.openshift.io/{{ .ClusterProfile }}: "true" spec: containers: - name: cluster-version-operator From 85b12d0ed506c272ab0e52caba61ede41736f470 Mon Sep 17 00:00:00 2001 From: Joel Speed Date: Wed, 21 Jan 2026 12:00:39 +0000 Subject: [PATCH 5/7] Rename enabledFeatureGates to enabledCVOFeatureGates --- pkg/cvo/availableupdates_test.go | 6 +++--- pkg/cvo/cvo.go | 23 +++++++++++++---------- pkg/cvo/cvo_featuregates_test.go | 4 ++-- pkg/cvo/cvo_scenarios_test.go | 18 +++++++++--------- pkg/cvo/cvo_test.go | 2 +- pkg/cvo/featuregate_integration_test.go | 2 +- pkg/cvo/status.go | 4 ++-- 7 files changed, 31 insertions(+), 28 deletions(-) diff --git a/pkg/cvo/availableupdates_test.go b/pkg/cvo/availableupdates_test.go index 38a620cadf..d9a07e88a0 100644 --- a/pkg/cvo/availableupdates_test.go +++ b/pkg/cvo/availableupdates_test.go @@ -236,7 +236,7 @@ func TestSyncAvailableUpdates(t *testing.T) { } expectedAvailableUpdates.RiskConditions = map[string][]metav1.Condition{"FourFiveSix": {{Type: "Applies", Status: "True", Reason: "Match"}}} - optr.enabledFeatureGates = featuregates.DefaultCvoGates("version") + optr.enabledCVOFeatureGates = featuregates.DefaultCvoGates("version") err := optr.syncAvailableUpdates(context.Background(), cvFixture) if err != nil { @@ -325,7 +325,7 @@ func TestSyncAvailableUpdates_ConditionalUpdateRecommendedConditions(t *testing. tc.modifyOriginalState(optr) tc.modifyCV(cv, fixture.expectedConditionalUpdates[0]) - optr.enabledFeatureGates = featuregates.DefaultCvoGates("version") + optr.enabledCVOFeatureGates = featuregates.DefaultCvoGates("version") err := optr.syncAvailableUpdates(context.Background(), cv) if err != nil { @@ -820,7 +820,7 @@ func TestSyncAvailableUpdatesDesiredUpdate(t *testing.T) { cv := cvFixture.DeepCopy() cv.Spec.DesiredUpdate = tt.args.desiredUpdate - optr.enabledFeatureGates = featuregates.DefaultCvoGates("version") + optr.enabledCVOFeatureGates = featuregates.DefaultCvoGates("version") if err := optr.syncAvailableUpdates(context.Background(), cv); err != nil { t.Fatalf("syncAvailableUpdates() unexpected error: %v", err) } diff --git a/pkg/cvo/cvo.go b/pkg/cvo/cvo.go index 2e5a1ea3fe..b9c0abb7b9 100644 --- a/pkg/cvo/cvo.go +++ b/pkg/cvo/cvo.go @@ -180,19 +180,22 @@ type Operator struct { // to select the manifests that will be applied in the cluster. The starting value cannot be changed in the executing // CVO but the featurechangestopper controller will detect a feature set change in the cluster and shutdown the CVO. // Enforcing featuresets is a standard GA CVO behavior that supports the feature gating functionality across the whole - // cluster, as opposed to the enabledFeatureGates which controls what gated behaviors of CVO itself are enabled by + // cluster, as opposed to the enabledCVOFeatureGates which controls what gated behaviors of CVO itself are enabled by // the cluster feature gates. // See: https://github.com/openshift/enhancements/blob/master/enhancements/update/cvo-techpreview-manifests.md requiredFeatureSet configv1.FeatureSet - // enabledFeatureGates is the checker for what gated CVO behaviors are enabled or disabled by specific cluster-level + // enabledCVOFeatureGates is the checker for what gated CVO behaviors are enabled or disabled by specific cluster-level // feature gates. It allows multiplexing the cluster-level feature gates to more granular CVO-level gates. Similarly - // to the requiredFeatureSet, the enabledFeatureGates cannot be changed in the executing CVO but the + // to the requiredFeatureSet, the enabledCVOFeatureGates cannot be changed in the executing CVO but the // featurechangestopper controller will detect when cluster feature gate config changes and shutdown the CVO. - enabledFeatureGates featuregates.CvoGateChecker + enabledCVOFeatureGates featuregates.CvoGateChecker // featureGatesMutex protects access to enabledManifestFeatureGates - featureGatesMutex sync.RWMutex + featureGatesMutex sync.RWMutex + // enabledManifestFeatureGates is the set of feature gates that are currently enabled for the manifests that are applied to the cluster. + // This is the full set of enabled feature gates extracted from the FeatureGate object. + // We use this set as a filter to determine which of the manifests from the payload should or should not be applied to the cluster. enabledManifestFeatureGates sets.Set[string] clusterProfile string @@ -267,7 +270,7 @@ func New( injectClusterIdIntoPromQL: injectClusterIdIntoPromQL, requiredFeatureSet: featureSet, - enabledFeatureGates: cvoGates, + enabledCVOFeatureGates: cvoGates, enabledManifestFeatureGates: startingEnabledManifestFeatureGates, alwaysEnableCapabilities: alwaysEnableCapabilities, @@ -1159,7 +1162,7 @@ func (optr *Operator) extractEnabledGates(featureGate *configv1.FeatureGate) set currentVersion := optr.currentVersion().Version if currentVersion == "" { klog.Warningf("Payload has not been initialized yet, using the operator version %s", optr.enabledCVOFeatureGates.DesiredVersion()) - currentVersion = optr.enabledFeatureGates.DesiredVersion() + currentVersion = optr.enabledCVOFeatureGates.DesiredVersion() } return featuregates.ExtractEnabledGates(featureGate, currentVersion) @@ -1167,10 +1170,10 @@ func (optr *Operator) extractEnabledGates(featureGate *configv1.FeatureGate) set // shouldReconcileCVOConfiguration returns whether the CVO should reconcile its configuration using the API server. // -// enabledFeatureGates must be initialized before the function is called. +// enabledCVOFeatureGates must be initialized before the function is called. func (optr *Operator) shouldReconcileCVOConfiguration() bool { // The relevant CRD and CR are not applied in HyperShift, which configures the CVO via a configuration file - return optr.enabledFeatureGates.CVOConfiguration() && !optr.hypershift + return optr.enabledCVOFeatureGates.CVOConfiguration() && !optr.hypershift } // shouldReconcileAcceptRisks returns whether the CVO should reconcile spec.desiredUpdate.acceptRisks and populate the @@ -1179,5 +1182,5 @@ func (optr *Operator) shouldReconcileCVOConfiguration() bool { // enabledFeatureGates must be initialized before the function is called. func (optr *Operator) shouldReconcileAcceptRisks() bool { // HyperShift will be supported later if needed - return optr.enabledFeatureGates.AcceptRisks() && !optr.hypershift + return optr.enabledCVOFeatureGates.AcceptRisks() && !optr.hypershift } diff --git a/pkg/cvo/cvo_featuregates_test.go b/pkg/cvo/cvo_featuregates_test.go index d4d5e648fa..b9e5da6aaa 100644 --- a/pkg/cvo/cvo_featuregates_test.go +++ b/pkg/cvo/cvo_featuregates_test.go @@ -76,7 +76,7 @@ func TestOperator_extractEnabledGates(t *testing.T) { t.Run(tt.name, func(t *testing.T) { optr := &Operator{ release: tt.release, - enabledFeatureGates: fakeRiFlags{ + enabledCVOFeatureGates: fakeRiFlags{ desiredVersion: tt.release.Version, }, } @@ -150,7 +150,7 @@ func TestOperator_updateEnabledFeatureGates(t *testing.T) { optr := &Operator{ enabledManifestFeatureGates: sets.New[string]("oldgate"), release: configv1.Release{Version: "4.14.0"}, - enabledFeatureGates: fakeRiFlags{ + enabledCVOFeatureGates: fakeRiFlags{ desiredVersion: "4.14.0", }, } diff --git a/pkg/cvo/cvo_scenarios_test.go b/pkg/cvo/cvo_scenarios_test.go index 11581c4f02..3cb6556a79 100644 --- a/pkg/cvo/cvo_scenarios_test.go +++ b/pkg/cvo/cvo_scenarios_test.go @@ -115,15 +115,15 @@ func setupCVOTest(payloadDir string) (*Operator, map[string]apiruntime.Object, * } o := &Operator{ - namespace: "test", - name: "version", - queue: workqueue.NewTypedRateLimitingQueueWithConfig[any](workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "cvo-loop-test"}), - client: client, - enabledFeatureGates: featuregates.DefaultCvoGates("version"), - cvLister: &clientCVLister{client: client}, - exclude: "exclude-test", - eventRecorder: record.NewFakeRecorder(100), - clusterProfile: payload.DefaultClusterProfile, + namespace: "test", + name: "version", + queue: workqueue.NewTypedRateLimitingQueueWithConfig[any](workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "cvo-loop-test"}), + client: client, + enabledCVOFeatureGates: featuregates.DefaultCvoGates("version"), + cvLister: &clientCVLister{client: client}, + exclude: "exclude-test", + eventRecorder: record.NewFakeRecorder(100), + clusterProfile: payload.DefaultClusterProfile, } dynamicScheme := apiruntime.NewScheme() diff --git a/pkg/cvo/cvo_test.go b/pkg/cvo/cvo_test.go index 33b568a8bc..6ea9c2bb7f 100644 --- a/pkg/cvo/cvo_test.go +++ b/pkg/cvo/cvo_test.go @@ -2275,7 +2275,7 @@ func TestOperator_sync(t *testing.T) { optr.configSync = &fakeSyncRecorder{Returns: expectStatus} } optr.eventRecorder = record.NewFakeRecorder(100) - optr.enabledFeatureGates = featuregates.DefaultCvoGates("version") + optr.enabledCVOFeatureGates = featuregates.DefaultCvoGates("version") ctx := context.Background() err := optr.sync(ctx, optr.queueKey()) diff --git a/pkg/cvo/featuregate_integration_test.go b/pkg/cvo/featuregate_integration_test.go index f0545f4f10..45248456bb 100644 --- a/pkg/cvo/featuregate_integration_test.go +++ b/pkg/cvo/featuregate_integration_test.go @@ -164,7 +164,7 @@ func TestFeatureGateEventHandling(t *testing.T) { // Create a simple operator with feature gate management capabilities optr := &Operator{ release: configv1.Release{Version: "4.14.0"}, - enabledFeatureGates: fakeRiFlags{ + enabledCVOFeatureGates: fakeRiFlags{ desiredVersion: "4.14.0", }, } diff --git a/pkg/cvo/status.go b/pkg/cvo/status.go index ea97e3cce4..815b7c73f2 100644 --- a/pkg/cvo/status.go +++ b/pkg/cvo/status.go @@ -164,7 +164,7 @@ func (optr *Operator) syncStatus(ctx context.Context, original, config *configv1 cvUpdated := false // update the config with the latest available updates - if updated := optr.getAvailableUpdates().NeedsUpdate(config, optr.enabledFeatureGates.StatusReleaseArchitecture()); updated != nil { + if updated := optr.getAvailableUpdates().NeedsUpdate(config, optr.enabledCVOFeatureGates.StatusReleaseArchitecture()); updated != nil { cvUpdated = true config = updated } @@ -177,7 +177,7 @@ func (optr *Operator) syncStatus(ctx context.Context, original, config *configv1 original = config.DeepCopy() } - updateClusterVersionStatus(&config.Status, status, optr.release, optr.getAvailableUpdates, optr.enabledFeatureGates, validationErrs, optr.shouldReconcileAcceptRisks) + updateClusterVersionStatus(&config.Status, status, optr.release, optr.getAvailableUpdates, optr.enabledCVOFeatureGates, validationErrs, optr.shouldReconcileAcceptRisks) if klog.V(6).Enabled() { klog.Infof("Apply config: %s", cmp.Diff(original, config)) From 84a0a3aa352e36997d7ff7c7374f18c052d1f9b2 Mon Sep 17 00:00:00 2001 From: Joel Speed Date: Fri, 23 Jan 2026 11:13:32 +0000 Subject: [PATCH 6/7] Adjust syncPayload handling to explicit switch on various changes --- pkg/cvo/sync_worker.go | 257 +++++++++++++++++++++-------------------- 1 file changed, 132 insertions(+), 125 deletions(-) diff --git a/pkg/cvo/sync_worker.go b/pkg/cvo/sync_worker.go index aa48d312fa..8c94b1917f 100644 --- a/pkg/cvo/sync_worker.go +++ b/pkg/cvo/sync_worker.go @@ -296,19 +296,19 @@ func (w *SyncWorker) syncPayload(ctx context.Context, work *SyncWork) ([]configv // cache the payload until the release image changes validPayload := w.payload - previousEnabledFeatureGates := w.status.EnabledFeatureGates - if !work.EnabledFeatureGates.Equal(previousEnabledFeatureGates) { + switch { + case validPayload == nil: + klog.V(2).Info("Loading initial payload") + case !equalUpdate(configv1.Update{Image: validPayload.Release.Image}, configv1.Update{Image: desired.Image}): + klog.V(2).Info("Loading payload due to desired image not being equal to the current one.") + case !work.EnabledFeatureGates.Equal(w.status.EnabledFeatureGates): // When the feature gates change, we must reload the payload. // Loading the payload filters out files that didn't match the previous set of feature gates, // this means now, additional files may match the new set of feature gates and need to be included. // Some files in the current payload may no longer match the new set of feature gates and need to be excluded, // though these ones are already excluded when apply calls Include on the manifests. - klog.V(2).Infof("Enabled feature gates changed from %v to %v, forcing a payload refresh", previousEnabledFeatureGates, work.EnabledFeatureGates) - w.payload = nil - } - - if validPayload != nil && validPayload.Release.Image == desired.Image { - + klog.V(2).Infof("Enabled feature gates changed from %v to %v, forcing a payload refresh", w.status.EnabledFeatureGates, work.EnabledFeatureGates) + case validPayload.Release.Image == desired.Image: // reset payload status to currently loaded payload if it no longer applies to desired target if !reporter.ValidPayloadStatus(desired) { klog.V(2).Info("Resetting payload status to currently loaded payload.") @@ -321,144 +321,151 @@ func (w *SyncWorker) syncPayload(ctx context.Context, work *SyncWork) ([]configv LastTransitionTime: time.Now(), }) } + // possibly complain here if Version, etc. diverges from the payload content return implicitlyEnabledCaps, nil - } else if validPayload == nil || !equalUpdate(configv1.Update{Image: validPayload.Release.Image}, configv1.Update{Image: desired.Image}) { - cvoObjectRef := &corev1.ObjectReference{APIVersion: "config.openshift.io/v1", Kind: "ClusterVersion", Name: "version", Namespace: "openshift-cluster-version"} - msg := fmt.Sprintf("Retrieving and verifying payload version=%q image=%q", desired.Version, desired.Image) - w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeNormal, "RetrievePayload", msg) + } + + // The remainder of this logic is for loading a new payload. + // Any filtering as to not needing to reload the payload should be done in the switch before this point. + + cvoObjectRef := &corev1.ObjectReference{APIVersion: "config.openshift.io/v1", Kind: "ClusterVersion", Name: "version", Namespace: "openshift-cluster-version"} + msg := fmt.Sprintf("Retrieving and verifying payload version=%q image=%q", desired.Version, desired.Image) + w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeNormal, "RetrievePayload", msg) + reporter.ReportPayload(LoadPayloadStatus{ + Step: "RetrievePayload", + Message: msg, + Update: desired, + LastTransitionTime: time.Now(), + }) + + // syncPayload executes while locked, but RetrievePayload is a potentially long-running operation + // which does not need the lock, so holding it may block other loops (mainly the apply loop) from + // execution + w.lock.Unlock() + info, err := w.retriever.RetrievePayload(ctx, work.Desired) + w.lock.Lock() + if err != nil { + msg := fmt.Sprintf("Retrieving payload failed version=%q image=%q failure=%s", desired.Version, desired.Image, strings.ReplaceAll(unwrappedErrorAggregate(err), "\n", " // ")) + w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeWarning, "RetrievePayloadFailed", msg) + msg = fmt.Sprintf("Retrieving payload failed version=%q image=%q failure=%s", desired.Version, desired.Image, err) reporter.ReportPayload(LoadPayloadStatus{ + Failure: err, Step: "RetrievePayload", Message: msg, Update: desired, + Local: info.Local, LastTransitionTime: time.Now(), }) + return nil, err + } - // syncPayload executes while locked, but RetrievePayload is a potentially long-running operation - // which does not need the lock, so holding it may block other loops (mainly the apply loop) from - // execution - w.lock.Unlock() - info, err := w.retriever.RetrievePayload(ctx, work.Desired) - w.lock.Lock() - if err != nil { - msg := fmt.Sprintf("Retrieving payload failed version=%q image=%q failure=%s", desired.Version, desired.Image, strings.ReplaceAll(unwrappedErrorAggregate(err), "\n", " // ")) - w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeWarning, "RetrievePayloadFailed", msg) - msg = fmt.Sprintf("Retrieving payload failed version=%q image=%q failure=%s", desired.Version, desired.Image, err) - reporter.ReportPayload(LoadPayloadStatus{ - Failure: err, - Step: "RetrievePayload", - Message: msg, - Update: desired, - Local: info.Local, - LastTransitionTime: time.Now(), - }) - return nil, err - } - acceptedRisksMsg := "" - if info.VerificationError != nil { - acceptedRisksMsg = unwrappedErrorAggregate(info.VerificationError) - w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeWarning, "RetrievePayload", acceptedRisksMsg) - } - - w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeNormal, "LoadPayload", "Loading payload version=%q image=%q", desired.Version, desired.Image) - - // Capability filtering is not done here since unknown capabilities are allowed - // during updated payload load and enablement checking only occurs during apply. - payloadUpdate, err := payload.LoadUpdate(info.Directory, desired.Image, w.exclude, string(w.requiredFeatureSet), w.clusterProfile, nil, work.EnabledFeatureGates) - - if err != nil { - msg := fmt.Sprintf("Loading payload failed version=%q image=%q failure=%v", desired.Version, desired.Image, err) - w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeWarning, "LoadPayloadFailed", msg) - reporter.ReportPayload(LoadPayloadStatus{ - Failure: err, - Step: "LoadPayload", - Message: msg, - Verified: info.Verified, - Local: info.Local, - Update: desired, - LastTransitionTime: time.Now(), - }) - return nil, err - } + acceptedRisksMsg := "" + if info.VerificationError != nil { + acceptedRisksMsg = unwrappedErrorAggregate(info.VerificationError) + w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeWarning, "RetrievePayload", acceptedRisksMsg) + } - payloadUpdate.VerifiedImage = info.Verified - payloadUpdate.LoadedAt = time.Now() + w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeNormal, "LoadPayload", "Loading payload version=%q image=%q", desired.Version, desired.Image) - if work.Desired.Version == "" { - work.Desired.Version = payloadUpdate.Release.Version - desired.Version = payloadUpdate.Release.Version - } else if payloadUpdate.Release.Version != work.Desired.Version { - err = fmt.Errorf("release image version %s does not match the expected upstream version %s", payloadUpdate.Release.Version, work.Desired.Version) - msg := fmt.Sprintf("Verifying payload failed version=%q image=%q failure=%v", work.Desired.Version, work.Desired.Image, err) - w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeWarning, "VerifyPayloadVersionFailed", msg) - reporter.ReportPayload(LoadPayloadStatus{ - Failure: err, - Step: "VerifyPayloadVersion", - Message: msg, - Verified: info.Verified, - Local: info.Local, - Update: desired, - LastTransitionTime: time.Now(), - }) - return nil, err - } + // Capability filtering is not done here since unknown capabilities are allowed + // during updated payload load and enablement checking only occurs during apply. + payloadUpdate, err := payload.LoadUpdate(info.Directory, desired.Image, w.exclude, string(w.requiredFeatureSet), w.clusterProfile, nil, work.EnabledFeatureGates) + if err != nil { + msg := fmt.Sprintf("Loading payload failed version=%q image=%q failure=%v", desired.Version, desired.Image, err) + w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeWarning, "LoadPayloadFailed", msg) + reporter.ReportPayload(LoadPayloadStatus{ + Failure: err, + Step: "LoadPayload", + Message: msg, + Verified: info.Verified, + Local: info.Local, + Update: desired, + LastTransitionTime: time.Now(), + }) + return nil, err + } - // need to make sure the payload is only set when the preconditions have been successful - if len(w.preconditions) == 0 { - klog.V(2).Info("No preconditions configured.") - } else if info.Local { - klog.V(2).Info("Skipping preconditions for a local operator image payload.") - } else { - if block, err := precondition.Summarize(w.preconditions.RunAll(ctx, precondition.ReleaseContext{ - DesiredVersion: payloadUpdate.Release.Version, - }), work.Desired.Force); err != nil { - klog.V(2).Infof("Precondition error (force %t, block %t): %v", work.Desired.Force, block, err) - if block { - msg := fmt.Sprintf("Preconditions failed for payload loaded version=%q image=%q: %v", desired.Version, desired.Image, err) - w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeWarning, "PreconditionBlock", msg) - reporter.ReportPayload(LoadPayloadStatus{ - Failure: err, - Step: "PreconditionChecks", - Message: msg, - Verified: info.Verified, - Local: info.Local, - Update: desired, - LastTransitionTime: time.Now(), - }) - return nil, err - } else { - w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeWarning, "PreconditionWarn", "precondition warning for payload loaded version=%q image=%q: %v", desired.Version, desired.Image, err) + payloadUpdate.VerifiedImage = info.Verified + payloadUpdate.LoadedAt = time.Now() - if acceptedRisksMsg == "" { - acceptedRisksMsg = err.Error() - } else { - acceptedRisksMsg = fmt.Sprintf("%s\n%s", acceptedRisksMsg, err.Error()) - } - } - } - w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeNormal, "PreconditionsPassed", "preconditions passed for payload loaded version=%q image=%q", desired.Version, desired.Image) - } - if w.payload != nil { - implicitlyEnabledCaps = capability.SortedList(payload.GetImplicitlyEnabledCapabilities(payloadUpdate.Manifests, w.payload.Manifests, - work.Capabilities, work.EnabledFeatureGates)) - } - w.payload = payloadUpdate - msg = fmt.Sprintf("Payload loaded version=%q image=%q architecture=%q", desired.Version, desired.Image, - payloadUpdate.Architecture) - w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeNormal, "PayloadLoaded", msg) + if work.Desired.Version == "" { + work.Desired.Version = payloadUpdate.Release.Version + desired.Version = payloadUpdate.Release.Version + } else if payloadUpdate.Release.Version != work.Desired.Version { + err = fmt.Errorf("release image version %s does not match the expected upstream version %s", payloadUpdate.Release.Version, work.Desired.Version) + msg := fmt.Sprintf("Verifying payload failed version=%q image=%q failure=%v", work.Desired.Version, work.Desired.Image, err) + w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeWarning, "VerifyPayloadVersionFailed", msg) reporter.ReportPayload(LoadPayloadStatus{ - Failure: nil, - Step: "PayloadLoaded", + Failure: err, + Step: "VerifyPayloadVersion", Message: msg, - AcceptedRisks: acceptedRisksMsg, Verified: info.Verified, Local: info.Local, Update: desired, LastTransitionTime: time.Now(), }) - klog.V(2).Infof("Payload loaded from %s with hash %s, architecture %s", desired.Image, payloadUpdate.ManifestHash, - payloadUpdate.Architecture) + return nil, err + } + + // need to make sure the payload is only set when the preconditions have been successful + if len(w.preconditions) == 0 { + klog.V(2).Info("No preconditions configured.") + } else if info.Local { + klog.V(2).Info("Skipping preconditions for a local operator image payload.") + } else { + if block, err := precondition.Summarize(w.preconditions.RunAll(ctx, precondition.ReleaseContext{ + DesiredVersion: payloadUpdate.Release.Version, + }), work.Desired.Force); err != nil { + klog.V(2).Infof("Precondition error (force %t, block %t): %v", work.Desired.Force, block, err) + if block { + msg := fmt.Sprintf("Preconditions failed for payload loaded version=%q image=%q: %v", desired.Version, desired.Image, err) + w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeWarning, "PreconditionBlock", msg) + reporter.ReportPayload(LoadPayloadStatus{ + Failure: err, + Step: "PreconditionChecks", + Message: msg, + Verified: info.Verified, + Local: info.Local, + Update: desired, + LastTransitionTime: time.Now(), + }) + return nil, err + } else { + w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeWarning, "PreconditionWarn", "precondition warning for payload loaded version=%q image=%q: %v", desired.Version, desired.Image, err) + + if acceptedRisksMsg == "" { + acceptedRisksMsg = err.Error() + } else { + acceptedRisksMsg = fmt.Sprintf("%s\n%s", acceptedRisksMsg, err.Error()) + } + } + } + w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeNormal, "PreconditionsPassed", "preconditions passed for payload loaded version=%q image=%q", desired.Version, desired.Image) + } + + if w.payload != nil { + implicitlyEnabledCaps = capability.SortedList(payload.GetImplicitlyEnabledCapabilities(payloadUpdate.Manifests, w.payload.Manifests, + work.Capabilities, work.EnabledFeatureGates)) } + + w.payload = payloadUpdate + msg = fmt.Sprintf("Payload loaded version=%q image=%q architecture=%q", desired.Version, desired.Image, + payloadUpdate.Architecture) + w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeNormal, "PayloadLoaded", msg) + reporter.ReportPayload(LoadPayloadStatus{ + Failure: nil, + Step: "PayloadLoaded", + Message: msg, + AcceptedRisks: acceptedRisksMsg, + Verified: info.Verified, + Local: info.Local, + Update: desired, + LastTransitionTime: time.Now(), + }) + klog.V(2).Infof("Payload loaded from %s with hash %s, architecture %s", desired.Image, payloadUpdate.ManifestHash, + payloadUpdate.Architecture) + return implicitlyEnabledCaps, nil } From f09b8116357edc0d9e5cbe43eb2583a9e945b483 Mon Sep 17 00:00:00 2001 From: David Hurta Date: Mon, 26 Jan 2026 18:24:46 +0100 Subject: [PATCH 7/7] test: Add CVO scenario for feature gate based inclusion Co-Authored-By: Claude Sonnet 4.5 --- pkg/cvo/cvo_scenarios_test.go | 344 ++++++++++++++++++ .../featuregatetest/manifests/.gitkeep | 0 .../0000_10_always-included.yaml | 9 + .../0000_20_experimental-feature.yaml | 10 + .../0000_30_legacy-excluded.yaml | 10 + .../release-manifests/image-references | 4 + .../release-manifests/release-metadata | 9 + 7 files changed, 386 insertions(+) create mode 100644 pkg/cvo/testdata/featuregatetest/manifests/.gitkeep create mode 100644 pkg/cvo/testdata/featuregatetest/release-manifests/0000_10_always-included.yaml create mode 100644 pkg/cvo/testdata/featuregatetest/release-manifests/0000_20_experimental-feature.yaml create mode 100644 pkg/cvo/testdata/featuregatetest/release-manifests/0000_30_legacy-excluded.yaml create mode 100644 pkg/cvo/testdata/featuregatetest/release-manifests/image-references create mode 100644 pkg/cvo/testdata/featuregatetest/release-manifests/release-metadata diff --git a/pkg/cvo/cvo_scenarios_test.go b/pkg/cvo/cvo_scenarios_test.go index 3cb6556a79..8b5a122a81 100644 --- a/pkg/cvo/cvo_scenarios_test.go +++ b/pkg/cvo/cvo_scenarios_test.go @@ -4045,6 +4045,350 @@ func TestCVO_VerifyUpdatingPayloadState(t *testing.T) { } } +// TestCVO_FeatureGateManifestInclusion tests that manifest inclusion changes dynamically +// when feature gates are updated, triggering payload refresh and re-filtering of manifests. +func TestCVO_FeatureGateManifestInclusion(t *testing.T) { + o, cvs, client, _, shutdownFn := setupCVOTest("testdata/featuregatetest") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + defer shutdownFn() + worker := o.configSync.(*SyncWorker) + go worker.Start(ctx, 1) + + // Step 1: Start with no feature gates enabled + // Expected manifests: always-included (no annotation), legacy-excluded (requires -LegacyFeature, which is not enabled) + // NOT included: experimental-feature (requires ExperimentalFeature) + o.release.Image = "image/image:1" + o.release.Version = "1.0.0-abc" + desired := configv1.Release{Version: "1.0.0-abc", Image: "image/image:1"} + uid, _ := uuid.NewRandom() + clusterUID := configv1.ClusterID(uid.String()) + cvs["version"] = &configv1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "version", + }, + Spec: configv1.ClusterVersionSpec{ + ClusterID: clusterUID, + Channel: "fast", + }, + } + + // Sync with no feature gates + client.ClearActions() + err := o.sync(ctx, o.queueKey()) + if err != nil { + t.Fatal(err) + } + + // Wait for payload to load + verifyAllStatus(t, worker.StatusCh(), + SyncWorkerStatus{ + Actual: desired, + loadPayloadStatus: LoadPayloadStatus{ + Step: "RetrievePayload", + Message: "Retrieving and verifying payload version=\"1.0.0-abc\" image=\"image/image:1\"", + LastTransitionTime: time.Unix(1, 0), + Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, + }, + EnabledFeatureGates: sets.New[string](), + }, + SyncWorkerStatus{ + Actual: desired, + LastProgress: time.Unix(1, 0), + loadPayloadStatus: LoadPayloadStatus{ + Step: "PayloadLoaded", + Message: "Payload loaded version=\"1.0.0-abc\" image=\"image/image:1\" architecture=\"" + architecture + "\"", + LastTransitionTime: time.Unix(2, 0), + Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, + }, + EnabledFeatureGates: sets.New[string](), + }, + SyncWorkerStatus{ + Total: 2, // only always-included and legacy-excluded + Initial: true, + VersionHash: "YAJ_K7RyH7U=", Architecture: architecture, + Actual: configv1.Release{ + Version: "1.0.0-abc", + Image: "image/image:1", + URL: "https://example.com/v1.0.0-abc", + }, + LastProgress: time.Unix(2, 0), + loadPayloadStatus: LoadPayloadStatus{ + Step: "PayloadLoaded", + Message: "Payload loaded version=\"1.0.0-abc\" image=\"image/image:1\" architecture=\"" + architecture + "\"", + LastTransitionTime: time.Unix(3, 0), + Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, + }, + CapabilitiesStatus: CapabilityStatus{ + Status: configv1.ClusterVersionCapabilitiesStatus{ + EnabledCapabilities: sortedCaps, + KnownCapabilities: sortedKnownCaps, + }, + }, + EnabledFeatureGates: sets.New[string](), + }, + ) + + // Wait for Step 1 to complete + waitForStatusCompleted(t, worker) + + // Verify 2 manifests in payload + if worker.payload == nil { + t.Fatal("Expected payload to be loaded") + } + if len(worker.payload.Manifests) != 2 { + t.Fatalf("Expected 2 manifests (without ExperimentalFeature), got %d", len(worker.payload.Manifests)) + } + + // Step 2: Enable ExperimentalFeature gate + // Expected manifests: always-included, legacy-excluded, AND experimental-feature + + // Clear any pending status updates from Step 1 + clearAllStatus(t, worker.StatusCh()) + + o.updateEnabledFeatureGates(&configv1.FeatureGate{ + Status: configv1.FeatureGateStatus{ + FeatureGates: []configv1.FeatureGateDetails{ + { + Version: "1.0.0-abc", + Enabled: []configv1.FeatureGateAttributes{ + {Name: "ExperimentalFeature"}, + }, + }, + }, + }, + }) + + // Trigger another sync - this should cause payload refresh + client.ClearActions() + err = o.sync(ctx, o.queueKey()) + if err != nil { + t.Fatal(err) + } + + // Verify feature gates changed and payload was refreshed + // Note: updateLoadStatus preserves apply status fields (Done, Total, Completed, Initial, VersionHash, LastProgress) + // from the previous status, so these will have Step 1's completed values + verifyAllStatus(t, worker.StatusCh(), + SyncWorkerStatus{ + Done: 2, + Total: 2, + Completed: 1, + Reconciling: true, + Initial: false, + VersionHash: "YAJ_K7RyH7U=", + Architecture: architecture, + Actual: configv1.Release{ + Version: "1.0.0-abc", + Image: "image/image:1", + URL: "https://example.com/v1.0.0-abc", + }, + LastProgress: time.Unix(1, 0), + loadPayloadStatus: LoadPayloadStatus{ + Step: "RetrievePayload", + Message: "Retrieving and verifying payload version=\"1.0.0-abc\" image=\"image/image:1\"", + LastTransitionTime: time.Unix(1, 0), + Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, + }, + CapabilitiesStatus: CapabilityStatus{ + Status: configv1.ClusterVersionCapabilitiesStatus{ + EnabledCapabilities: sortedCaps, + KnownCapabilities: sortedKnownCaps, + }, + }, + EnabledFeatureGates: sets.New[string](), + }, + SyncWorkerStatus{ + Done: 2, + Total: 2, + Completed: 1, + Reconciling: true, + Initial: false, + VersionHash: "YAJ_K7RyH7U=", + Architecture: architecture, + Actual: configv1.Release{ + Version: "1.0.0-abc", + Image: "image/image:1", + URL: "https://example.com/v1.0.0-abc", + }, + LastProgress: time.Unix(2, 0), + loadPayloadStatus: LoadPayloadStatus{ + Step: "PayloadLoaded", + Message: "Payload loaded version=\"1.0.0-abc\" image=\"image/image:1\" architecture=\"" + architecture + "\"", + LastTransitionTime: time.Unix(2, 0), + Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, + }, + CapabilitiesStatus: CapabilityStatus{ + Status: configv1.ClusterVersionCapabilitiesStatus{ + EnabledCapabilities: sortedCaps, + KnownCapabilities: sortedKnownCaps, + }, + }, + EnabledFeatureGates: sets.New[string](), + }, + SyncWorkerStatus{ + Total: 3, // now includes experimental-feature + Initial: false, + VersionHash: "yrh5CWG1KPI=", Architecture: architecture, + Actual: configv1.Release{ + Version: "1.0.0-abc", + Image: "image/image:1", + URL: "https://example.com/v1.0.0-abc", + }, + LastProgress: time.Unix(3, 0), + loadPayloadStatus: LoadPayloadStatus{ + Step: "PayloadLoaded", + Message: "Payload loaded version=\"1.0.0-abc\" image=\"image/image:1\" architecture=\"" + architecture + "\"", + LastTransitionTime: time.Unix(3, 0), + Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, + }, + CapabilitiesStatus: CapabilityStatus{ + Status: configv1.ClusterVersionCapabilitiesStatus{ + EnabledCapabilities: sortedCaps, + KnownCapabilities: sortedKnownCaps, + }, + ImplicitlyEnabledCaps: []configv1.ClusterVersionCapability{}, + }, + EnabledFeatureGates: sets.New[string]("ExperimentalFeature"), + }, + ) + + // Wait for Step 2 to complete + waitForStatusCompleted(t, worker) + + // Verify 3 manifests now (experimental-feature is now included) + if worker.payload == nil { + t.Fatal("Expected payload to be loaded") + } + if len(worker.payload.Manifests) != 3 { + t.Fatalf("Expected 3 manifests (with ExperimentalFeature), got %d", len(worker.payload.Manifests)) + } + + // Step 3: Enable LegacyFeature gate + // Expected manifests: always-included, experimental-feature + // NOT included: legacy-excluded (requires -LegacyFeature, but LegacyFeature is now enabled) + + // Clear any pending status updates from Step 2 + clearAllStatus(t, worker.StatusCh()) + + o.updateEnabledFeatureGates(&configv1.FeatureGate{ + Status: configv1.FeatureGateStatus{ + FeatureGates: []configv1.FeatureGateDetails{ + { + Version: "1.0.0-abc", + Enabled: []configv1.FeatureGateAttributes{ + {Name: "ExperimentalFeature"}, + {Name: "LegacyFeature"}, + }, + }, + }, + }, + }) + + // Trigger another sync + client.ClearActions() + err = o.sync(ctx, o.queueKey()) + if err != nil { + t.Fatal(err) + } + + // Verify feature gates changed and payload was refreshed again + // Note: updateLoadStatus preserves apply status fields from Step 2's completed state + verifyAllStatus(t, worker.StatusCh(), + SyncWorkerStatus{ + Done: 3, + Total: 3, + Completed: 2, + Reconciling: true, + Initial: false, + VersionHash: "yrh5CWG1KPI=", + Architecture: architecture, + Actual: configv1.Release{ + Version: "1.0.0-abc", + Image: "image/image:1", + URL: "https://example.com/v1.0.0-abc", + }, + LastProgress: time.Unix(1, 0), + loadPayloadStatus: LoadPayloadStatus{ + Step: "RetrievePayload", + Message: "Retrieving and verifying payload version=\"1.0.0-abc\" image=\"image/image:1\"", + LastTransitionTime: time.Unix(1, 0), + Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, + }, + CapabilitiesStatus: CapabilityStatus{ + Status: configv1.ClusterVersionCapabilitiesStatus{ + EnabledCapabilities: sortedCaps, + KnownCapabilities: sortedKnownCaps, + }, + }, + EnabledFeatureGates: sets.New[string]("ExperimentalFeature"), + }, + SyncWorkerStatus{ + Done: 3, + Total: 3, + Completed: 2, + Reconciling: true, + Initial: false, + VersionHash: "yrh5CWG1KPI=", + Architecture: architecture, + Actual: configv1.Release{ + Version: "1.0.0-abc", + Image: "image/image:1", + URL: "https://example.com/v1.0.0-abc", + }, + LastProgress: time.Unix(2, 0), + loadPayloadStatus: LoadPayloadStatus{ + Step: "PayloadLoaded", + Message: "Payload loaded version=\"1.0.0-abc\" image=\"image/image:1\" architecture=\"" + architecture + "\"", + LastTransitionTime: time.Unix(2, 0), + Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, + }, + CapabilitiesStatus: CapabilityStatus{ + Status: configv1.ClusterVersionCapabilitiesStatus{ + EnabledCapabilities: sortedCaps, + KnownCapabilities: sortedKnownCaps, + }, + }, + EnabledFeatureGates: sets.New[string]("ExperimentalFeature"), + }, + SyncWorkerStatus{ + Total: 2, // legacy-excluded is now excluded + Initial: false, + VersionHash: "ge54Uoy7v5o=", Architecture: architecture, + Actual: configv1.Release{ + Version: "1.0.0-abc", + Image: "image/image:1", + URL: "https://example.com/v1.0.0-abc", + }, + LastProgress: time.Unix(3, 0), + loadPayloadStatus: LoadPayloadStatus{ + Step: "PayloadLoaded", + Message: "Payload loaded version=\"1.0.0-abc\" image=\"image/image:1\" architecture=\"" + architecture + "\"", + LastTransitionTime: time.Unix(3, 0), + Update: configv1.Update{Version: "1.0.0-abc", Image: "image/image:1"}, + }, + CapabilitiesStatus: CapabilityStatus{ + Status: configv1.ClusterVersionCapabilitiesStatus{ + EnabledCapabilities: sortedCaps, + KnownCapabilities: sortedKnownCaps, + }, + ImplicitlyEnabledCaps: []configv1.ClusterVersionCapability{}, + }, + EnabledFeatureGates: sets.New[string]("ExperimentalFeature", "LegacyFeature"), + }, + ) + + // Verify 2 manifests now (legacy-excluded is now filtered out) + if worker.payload == nil { + t.Fatal("Expected payload to be loaded") + } + if len(worker.payload.Manifests) != 2 { + t.Fatalf("Expected 2 manifests (legacy-excluded should be filtered), got %d", len(worker.payload.Manifests)) + } +} + // verifyCVSingleUpdate ensures that the only object to be updated is a ClusterVersion type and it is updated only once func verifyCVSingleUpdate(t *testing.T, actions []clientgotesting.Action) { var count int diff --git a/pkg/cvo/testdata/featuregatetest/manifests/.gitkeep b/pkg/cvo/testdata/featuregatetest/manifests/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pkg/cvo/testdata/featuregatetest/release-manifests/0000_10_always-included.yaml b/pkg/cvo/testdata/featuregatetest/release-manifests/0000_10_always-included.yaml new file mode 100644 index 0000000000..d368069f63 --- /dev/null +++ b/pkg/cvo/testdata/featuregatetest/release-manifests/0000_10_always-included.yaml @@ -0,0 +1,9 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: always-included + namespace: default + annotations: + include.release.openshift.io/self-managed-high-availability: "true" +data: + key: "always-present" diff --git a/pkg/cvo/testdata/featuregatetest/release-manifests/0000_20_experimental-feature.yaml b/pkg/cvo/testdata/featuregatetest/release-manifests/0000_20_experimental-feature.yaml new file mode 100644 index 0000000000..5150c8b50d --- /dev/null +++ b/pkg/cvo/testdata/featuregatetest/release-manifests/0000_20_experimental-feature.yaml @@ -0,0 +1,10 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: experimental-feature + namespace: default + annotations: + include.release.openshift.io/self-managed-high-availability: "true" + release.openshift.io/feature-gate: "ExperimentalFeature" +data: + key: "experimental-data" diff --git a/pkg/cvo/testdata/featuregatetest/release-manifests/0000_30_legacy-excluded.yaml b/pkg/cvo/testdata/featuregatetest/release-manifests/0000_30_legacy-excluded.yaml new file mode 100644 index 0000000000..4b0d7b9650 --- /dev/null +++ b/pkg/cvo/testdata/featuregatetest/release-manifests/0000_30_legacy-excluded.yaml @@ -0,0 +1,10 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: legacy-excluded + namespace: default + annotations: + include.release.openshift.io/self-managed-high-availability: "true" + release.openshift.io/feature-gate: "-LegacyFeature" +data: + key: "new-implementation" diff --git a/pkg/cvo/testdata/featuregatetest/release-manifests/image-references b/pkg/cvo/testdata/featuregatetest/release-manifests/image-references new file mode 100644 index 0000000000..c9f98dd739 --- /dev/null +++ b/pkg/cvo/testdata/featuregatetest/release-manifests/image-references @@ -0,0 +1,4 @@ +kind: ImageStream +apiVersion: image.openshift.io/v1 +metadata: + name: 1.0.0-abc diff --git a/pkg/cvo/testdata/featuregatetest/release-manifests/release-metadata b/pkg/cvo/testdata/featuregatetest/release-manifests/release-metadata new file mode 100644 index 0000000000..e23830f39e --- /dev/null +++ b/pkg/cvo/testdata/featuregatetest/release-manifests/release-metadata @@ -0,0 +1,9 @@ +{ + "kind": "cincinnati-metadata-v0", + "version": "1.0.0-abc", + "previous": [], + "metadata": { + "description": "", + "url": "https://example.com/v1.0.0-abc" + } +} \ No newline at end of file