OCPSTRAT-2485: Introduce feature gate based inclusion/exclusion of manifests

bootstrap/bootstrap-pod.yaml

@@ -5,6 +5,8 @@ metadata:
   namespace: openshift-cluster-version
   labels:
     k8s-app: cluster-version-operator
+  annotations:
+    include.release.openshift.io/{{ .ClusterProfile }}: "true"
 spec:
   containers:
   - name: cluster-version-operator

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/openshift-eng/openshift-tests-extension v0.0.0-20250220212757-b9c4d98a0c45
 	github.com/openshift/api v0.0.0-20260116192047-6fb7fdae95fd
 	github.com/openshift/client-go v0.0.0-20260108185524-48f4ccfc4e13
-	github.com/openshift/library-go v0.0.0-20260108135436-db8dbd64c462
+	github.com/openshift/library-go v0.0.0-20260121132910-dc3a1c884c04
 	github.com/operator-framework/api v0.17.1
 	github.com/operator-framework/operator-lifecycle-manager v0.22.0
 	github.com/pkg/errors v0.9.1

go.sum

@@ -90,8 +90,8 @@ github.com/openshift/api v0.0.0-20260116192047-6fb7fdae95fd h1:Hpjq/55Qb0Gy65Rew
 github.com/openshift/api v0.0.0-20260116192047-6fb7fdae95fd/go.mod h1:d5uzF0YN2nQQFA0jIEWzzOZ+edmo6wzlGLvx5Fhz4uY=
 github.com/openshift/client-go v0.0.0-20260108185524-48f4ccfc4e13 h1:6rd4zSo2UaWQcAPZfHK9yzKVqH0BnMv1hqMzqXZyTds=
 github.com/openshift/client-go v0.0.0-20260108185524-48f4ccfc4e13/go.mod h1:YvOmPmV7wcJxpfhTDuFqqs2Xpb3M3ovsM6Qs/i2ptq4=
-github.com/openshift/library-go v0.0.0-20260108135436-db8dbd64c462 h1:zX9Od4Jg8sVmwQLwk6Vd+BX7tcyC/462FVvDdzHEPPk=
-github.com/openshift/library-go v0.0.0-20260108135436-db8dbd64c462/go.mod h1:nIzWQQE49XbiKizVnVOip9CEB7HJ0hoJwNi3g3YKnKc=
+github.com/openshift/library-go v0.0.0-20260121132910-dc3a1c884c04 h1:Fm9C8pT4l6VjpdqdhI1cBX9Y3D3S+rFxptVhCPBbMAI=
+github.com/openshift/library-go v0.0.0-20260121132910-dc3a1c884c04/go.mod h1:nIzWQQE49XbiKizVnVOip9CEB7HJ0hoJwNi3g3YKnKc=
 github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12 h1:AKx/w1qpS8We43bsRgf8Nll3CGlDHpr/WAXvuedTNZI=
 github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
 github.com/operator-framework/api v0.17.1 h1:J/6+Xj4IEV8C7hcirqUFwOiZAU3PbnJhWvB0/bB51c4=

hack/cluster-version-util/task_graph.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
+	"k8s.io/apimachinery/pkg/util/sets"
 
 	"github.com/openshift/cluster-version-operator/pkg/payload"
 )
@@ -30,7 +31,7 @@ func newTaskGraphCmd() *cobra.Command {
 
 func runTaskGraphCmd(cmd *cobra.Command, args []string) error {
 	manifestDir := args[0]
-	release, err := payload.LoadUpdate(manifestDir, "", "", "", payload.DefaultClusterProfile, nil)
+	release, err := payload.LoadUpdate(manifestDir, "", "", "", payload.DefaultClusterProfile, nil, sets.Set[string]{})
 	if err != nil {
 		return err
 	}

lib/manifest/manifest.go

@@ -33,6 +33,9 @@ type InclusionConfiguration struct {
 
 	// Platform, if non-nil, excludes CredentialsRequests manifests unless they match the infrastructure platform.
 	Platform *string
+
+	// EnabledFeatureGates excludes manifests with a feature gate requirement when the condition is not met.
+	EnabledFeatureGates sets.Set[string]
 }
 
 // GetImplicitlyEnabledCapabilities returns a set of capabilities that are implicitly enabled after a cluster update.
@@ -57,6 +60,7 @@ func GetImplicitlyEnabledCapabilities(
 			manifestInclusionConfiguration.Profile,
 			manifestInclusionConfiguration.Capabilities,
 			manifestInclusionConfiguration.Overrides,
+			manifestInclusionConfiguration.EnabledFeatureGates,
 			true,
 		)
 		// update manifest is enabled, no need to check
@@ -74,6 +78,7 @@ func GetImplicitlyEnabledCapabilities(
 				manifestInclusionConfiguration.Profile,
 				manifestInclusionConfiguration.Capabilities,
 				manifestInclusionConfiguration.Overrides,
+				manifestInclusionConfiguration.EnabledFeatureGates,
 				true,
 			); err != nil {
 				continue

pkg/cvo/availableupdates_test.go

@@ -236,7 +236,7 @@ func TestSyncAvailableUpdates(t *testing.T) {
 	}
 	expectedAvailableUpdates.RiskConditions = map[string][]metav1.Condition{"FourFiveSix": {{Type: "Applies", Status: "True", Reason: "Match"}}}
 
-	optr.enabledFeatureGates = featuregates.DefaultCvoGates("version")
+	optr.enabledCVOFeatureGates = featuregates.DefaultCvoGates("version")
 	err := optr.syncAvailableUpdates(context.Background(), cvFixture)
 
 	if err != nil {
@@ -325,7 +325,7 @@ func TestSyncAvailableUpdates_ConditionalUpdateRecommendedConditions(t *testing.
 			tc.modifyOriginalState(optr)
 			tc.modifyCV(cv, fixture.expectedConditionalUpdates[0])
 
-			optr.enabledFeatureGates = featuregates.DefaultCvoGates("version")
+			optr.enabledCVOFeatureGates = featuregates.DefaultCvoGates("version")
 			err := optr.syncAvailableUpdates(context.Background(), cv)
 
 			if err != nil {
@@ -820,7 +820,7 @@ func TestSyncAvailableUpdatesDesiredUpdate(t *testing.T) {
 
 			cv := cvFixture.DeepCopy()
 			cv.Spec.DesiredUpdate = tt.args.desiredUpdate
-			optr.enabledFeatureGates = featuregates.DefaultCvoGates("version")
+			optr.enabledCVOFeatureGates = featuregates.DefaultCvoGates("version")
 			if err := optr.syncAvailableUpdates(context.Background(), cv); err != nil {
 				t.Fatalf("syncAvailableUpdates() unexpected error: %v", err)
 			}

pkg/cvo/cvo.go

@@ -13,6 +13,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	informerscorev1 "k8s.io/client-go/informers/core/v1"
 	"k8s.io/client-go/kubernetes"
@@ -120,6 +121,7 @@ type Operator struct {
 	cmConfigLister        listerscorev1.ConfigMapNamespaceLister
 	cmConfigManagedLister listerscorev1.ConfigMapNamespaceLister
 	proxyLister           configlistersv1.ProxyLister
+	featureGateLister     configlistersv1.FeatureGateLister
 	cacheSynced           []cache.InformerSynced
 
 	// queue tracks applying updates to a cluster.
@@ -178,16 +180,23 @@ type Operator struct {
 	// to select the manifests that will be applied in the cluster. The starting value cannot be changed in the executing
 	// CVO but the featurechangestopper controller will detect a feature set change in the cluster and shutdown the CVO.
 	// Enforcing featuresets is a standard GA CVO behavior that supports the feature gating functionality across the whole
-	// cluster, as opposed to the enabledFeatureGates which controls what gated behaviors of CVO itself are enabled by
+	// cluster, as opposed to the enabledCVOFeatureGates which controls what gated behaviors of CVO itself are enabled by
 	// the cluster feature gates.
 	// See: https://github.com/openshift/enhancements/blob/master/enhancements/update/cvo-techpreview-manifests.md
 	requiredFeatureSet configv1.FeatureSet
 
-	// enabledFeatureGates is the checker for what gated CVO behaviors are enabled or disabled by specific cluster-level
+	// enabledCVOFeatureGates is the checker for what gated CVO behaviors are enabled or disabled by specific cluster-level
 	// feature gates. It allows multiplexing the cluster-level feature gates to more granular CVO-level gates. Similarly
-	// to the requiredFeatureSet, the enabledFeatureGates cannot be changed in the executing CVO but the
+	// to the requiredFeatureSet, the enabledCVOFeatureGates cannot be changed in the executing CVO but the
 	// featurechangestopper controller will detect when cluster feature gate config changes and shutdown the CVO.
-	enabledFeatureGates featuregates.CvoGateChecker
+	enabledCVOFeatureGates featuregates.CvoGateChecker
+
+	// featureGatesMutex protects access to enabledManifestFeatureGates
+	featureGatesMutex sync.RWMutex
+	// enabledManifestFeatureGates is the set of feature gates that are currently enabled for the manifests that are applied to the cluster.
+	// This is the full set of enabled feature gates extracted from the FeatureGate object.
+	// We use this set as a filter to determine which of the manifests from the payload should or should not be applied to the cluster.
+	enabledManifestFeatureGates sets.Set[string]
 
 	clusterProfile string
 	uid            types.UID
@@ -213,6 +222,7 @@ func New(
 	cmConfigManagedInformer informerscorev1.ConfigMapInformer,
 	proxyInformer configinformersv1.ProxyInformer,
 	operatorInformerFactory operatorexternalversions.SharedInformerFactory,
+	featureGateInformer configinformersv1.FeatureGateInformer,
 	client clientset.Interface,
 	kubeClient kubernetes.Interface,
 	operatorClient operatorclientset.Interface,
@@ -225,6 +235,7 @@ func New(
 	alwaysEnableCapabilities []configv1.ClusterVersionCapability,
 	featureSet configv1.FeatureSet,
 	cvoGates featuregates.CvoGateChecker,
+	startingEnabledManifestFeatureGates sets.Set[string],
 ) (*Operator, error) {
 	eventBroadcaster := record.NewBroadcaster()
 	eventBroadcaster.StartLogging(klog.Infof)
@@ -248,18 +259,19 @@ func New(
 		kubeClient:            kubeClient,
 		operatorClient:        operatorClient,
 		eventRecorder:         eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: namespace}),
-		queue:                 workqueue.NewTypedRateLimitingQueueWithConfig[any](workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "clusterversion"}),
-		availableUpdatesQueue: workqueue.NewTypedRateLimitingQueueWithConfig[any](workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "availableupdates"}),
-		upgradeableQueue:      workqueue.NewTypedRateLimitingQueueWithConfig[any](workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "upgradeable"}),
+		queue:                 workqueue.NewTypedRateLimitingQueueWithConfig(workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "clusterversion"}),
+		availableUpdatesQueue: workqueue.NewTypedRateLimitingQueueWithConfig(workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "availableupdates"}),
+		upgradeableQueue:      workqueue.NewTypedRateLimitingQueueWithConfig(workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "upgradeable"}),
 
 		hypershift:                hypershift,
 		exclude:                   exclude,
 		clusterProfile:            clusterProfile,
 		conditionRegistry:         standard.NewConditionRegistry(promqlTarget),
 		injectClusterIdIntoPromQL: injectClusterIdIntoPromQL,
 
-		requiredFeatureSet:  featureSet,
-		enabledFeatureGates: cvoGates,
+		requiredFeatureSet:          featureSet,
+		enabledCVOFeatureGates:      cvoGates,
+		enabledManifestFeatureGates: startingEnabledManifestFeatureGates,
 
 		alwaysEnableCapabilities: alwaysEnableCapabilities,
 	}
@@ -276,6 +288,9 @@ func New(
 	if _, err := coInformer.Informer().AddEventHandler(optr.clusterOperatorEventHandler()); err != nil {
 		return nil, err
 	}
+	if _, err := featureGateInformer.Informer().AddEventHandler(optr.featureGateEventHandler()); err != nil {
+		return nil, err
+	}
 
 	optr.coLister = coInformer.Lister()
 	optr.cacheSynced = append(optr.cacheSynced, coInformer.Informer().HasSynced)
@@ -287,6 +302,9 @@ func New(
 	optr.cmConfigLister = cmConfigInformer.Lister().ConfigMaps(internal.ConfigNamespace)
 	optr.cmConfigManagedLister = cmConfigManagedInformer.Lister().ConfigMaps(internal.ConfigManagedNamespace)
 
+	optr.featureGateLister = featureGateInformer.Lister()
+	optr.cacheSynced = append(optr.cacheSynced, featureGateInformer.Informer().HasSynced)
+
 	// make sure this is initialized after all the listers are initialized
 	optr.upgradeableChecks = optr.defaultUpgradeableChecks()
 
@@ -318,7 +336,7 @@ func (optr *Operator) LoadInitialPayload(ctx context.Context, restConfig *rest.C
 	}
 
 	update, err := payload.LoadUpdate(optr.defaultPayloadDir(), optr.release.Image, optr.exclude, string(optr.requiredFeatureSet),
-		optr.clusterProfile, configv1.KnownClusterVersionCapabilities)
+		optr.clusterProfile, configv1.KnownClusterVersionCapabilities, optr.getEnabledFeatureGates())
 
 	if err != nil {
 		return nil, fmt.Errorf("the local release contents are invalid - no current version can be determined from disk: %v", err)
@@ -779,7 +797,7 @@ func (optr *Operator) sync(ctx context.Context, key string) error {
 	}
 
 	// inform the config sync loop about our desired state
-	status := optr.configSync.Update(ctx, config.Generation, desired, config, state)
+	status := optr.configSync.Update(ctx, config.Generation, desired, config, state, optr.getEnabledFeatureGates())
 
 	// write cluster version status
 	return optr.syncStatus(ctx, original, config, status, errs)
@@ -1084,12 +1102,78 @@ func (optr *Operator) HTTPClient() (*http.Client, error) {
 	}, nil
 }
 
+// featureGateEventHandler handles changes to FeatureGate objects and updates the cluster feature gates
+func (optr *Operator) featureGateEventHandler() cache.ResourceEventHandler {
+	workQueueKey := optr.queueKey()
+	return cache.ResourceEventHandlerFuncs{
+		AddFunc: func(obj interface{}) {
+			if optr.updateEnabledFeatureGates(obj) {
+				optr.queue.Add(workQueueKey)
+			}
+		},
+		UpdateFunc: func(old, new interface{}) {
+			if optr.updateEnabledFeatureGates(new) {
+				optr.queue.Add(workQueueKey)
+			}
+		},
+	}
+}
+
+// updateEnabledFeatureGates updates the cluster feature gates based on a FeatureGate object.
+// Returns true or false based on whether or not the gates were actually updated.
+// This allows us to avoid unnecessary work if the gates have not changed.
+func (optr *Operator) updateEnabledFeatureGates(obj interface{}) bool {
+	featureGate, ok := obj.(*configv1.FeatureGate)
+	if !ok {
+		klog.Warningf("Expected FeatureGate object but got %T", obj)
+		return false
+	}
+
+	newGates := optr.extractEnabledGates(featureGate)
+
+	optr.featureGatesMutex.Lock()
+	defer optr.featureGatesMutex.Unlock()
+
+	// Check if gates actually changed to avoid unnecessary work
+	if !optr.enabledManifestFeatureGates.Equal(newGates) {
+
+		klog.V(2).Infof("Cluster feature gates changed from %v to %v",
+			sets.List(optr.enabledManifestFeatureGates), sets.List(newGates))
+
+		optr.enabledManifestFeatureGates = newGates
+		return true
+	}
+
+	return false
+}
+
+// getEnabledFeatureGates returns a copy of the current cluster feature gates for safe consumption
+func (optr *Operator) getEnabledFeatureGates() sets.Set[string] {
+	optr.featureGatesMutex.RLock()
+	defer optr.featureGatesMutex.RUnlock()
+
+	// Return a copy to prevent external modification
+	return optr.enabledManifestFeatureGates.Clone()
+}
+
+// extractEnabledGates extracts the list of enabled feature gates for the current cluster version
+func (optr *Operator) extractEnabledGates(featureGate *configv1.FeatureGate) sets.Set[string] {
+	// Find the feature gate details for the current loaded payload version.
+	currentVersion := optr.currentVersion().Version
+	if currentVersion == "" {
+		klog.Warningf("Payload has not been initialized yet, using the operator version %s", optr.enabledCVOFeatureGates.DesiredVersion())
+		currentVersion = optr.enabledCVOFeatureGates.DesiredVersion()
+	}
+
+	return featuregates.ExtractEnabledGates(featureGate, currentVersion)
+}
+
 // shouldReconcileCVOConfiguration returns whether the CVO should reconcile its configuration using the API server.
 //
-// enabledFeatureGates must be initialized before the function is called.
+// enabledCVOFeatureGates must be initialized before the function is called.
 func (optr *Operator) shouldReconcileCVOConfiguration() bool {
 	// The relevant CRD and CR are not applied in HyperShift, which configures the CVO via a configuration file
-	return optr.enabledFeatureGates.CVOConfiguration() && !optr.hypershift
+	return optr.enabledCVOFeatureGates.CVOConfiguration() && !optr.hypershift
 }
 
 // shouldReconcileAcceptRisks returns whether the CVO should reconcile spec.desiredUpdate.acceptRisks and populate the
@@ -1098,5 +1182,5 @@ func (optr *Operator) shouldReconcileCVOConfiguration() bool {
 // enabledFeatureGates must be initialized before the function is called.
 func (optr *Operator) shouldReconcileAcceptRisks() bool {
 	// HyperShift will be supported later if needed
-	return optr.enabledFeatureGates.AcceptRisks() && !optr.hypershift
+	return optr.enabledCVOFeatureGates.AcceptRisks() && !optr.hypershift
 }