Skip to content

Commit e9ba1bc

Browse files
JunAr7112JunAr7112
committed
Updated Red Hat mounts for air-gapped clusters
Signed-off-by: Arjun <agadiyar@nvidia.com>
1 parent 9aff6f5 commit e9ba1bc

4 files changed

Lines changed: 248 additions & 8 deletions

File tree

‎controllers/object_controls.go‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3704,10 +3704,15 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy
37043704

37053705
// set up subscription entitlements for RHEL(using K8s with a non-CRIO runtime) and SLES
37063706
if (osID == "rhel" && n.openshift == "" && n.runtime != gpuv1.CRIO) || osID == "sles" || osID == "sl-micro" {
3707-
n.logger.Info("Mounting subscriptions into the driver container", "OS", osID)
3708-
pathToVolumeSource, err := n.getSubscriptionPathsToVolumeSources()
3709-
if err != nil {
3710-
return fmt.Errorf("ERROR: failed to get path items for subscription entitlements: %v", err)
3707+
pathToVolumeSource := MountPathToVolumeSource{}
3708+
if config.Driver.RepoConfig != nil && config.Driver.RepoConfig.ConfigMapName != "" && osID == "rhel" {
3709+
n.logger.Info("Skipping host subscription mounts because repoConfig is enabled", "OS", osID)
3710+
} else {
3711+
n.logger.Info("Mounting subscriptions into the driver container", "OS", osID)
3712+
pathToVolumeSource, err = n.getSubscriptionPathsToVolumeSources()
3713+
if err != nil {
3714+
return fmt.Errorf("ERROR: failed to get path items for subscription entitlements: %w", err)
3715+
}
37113716
}
37123717

37133718
// sort host path volumes to ensure ordering is preserved when adding to pod spec

‎controllers/transforms_test.go‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package controllers
1818

1919
import (
2020
"path/filepath"
21+
"strings"
2122
"testing"
2223

2324
kata_v1alpha1 "github.com/NVIDIA/k8s-kata-manager/api/v1alpha1/config"
@@ -4347,6 +4348,115 @@ func TestTransformDriverWithAdditionalConfig(t *testing.T) {
43474348
}
43484349
}
43494350

4351+
func TestTransformDriverSubscriptionMounts(t *testing.T) {
4352+
repoConfigMap := &corev1.ConfigMap{
4353+
ObjectMeta: metav1.ObjectMeta{
4354+
Name: "test-repo-config",
4355+
Namespace: "test-ns",
4356+
},
4357+
Data: map[string]string{
4358+
"redhat.repo": "[test-repo]",
4359+
},
4360+
}
4361+
mockClient := fake.NewFakeClient(repoConfigMap)
4362+
4363+
testCases := []struct {
4364+
description string
4365+
osRelease string
4366+
osTag string
4367+
repoConfigEnabled bool
4368+
expectSubscriptionMounts bool
4369+
expectedSubscriptionHostMap map[string]corev1.HostPathType
4370+
}{
4371+
{
4372+
description: "rhel with repo config skips host subscription mounts",
4373+
osRelease: "rhel",
4374+
osTag: "rhel8.10",
4375+
repoConfigEnabled: true,
4376+
expectSubscriptionMounts: false,
4377+
},
4378+
{
4379+
description: "rhel without repo config mounts host subscription paths",
4380+
osRelease: "rhel",
4381+
osTag: "rhel8.10",
4382+
expectSubscriptionMounts: true,
4383+
expectedSubscriptionHostMap: map[string]corev1.HostPathType{
4384+
"/etc/pki/entitlement": corev1.HostPathDirectory,
4385+
"/etc/yum.repos.d/redhat.repo": corev1.HostPathFile,
4386+
"/etc/rhsm": corev1.HostPathDirectory,
4387+
},
4388+
},
4389+
}
4390+
4391+
for _, tc := range testCases {
4392+
t.Run(tc.description, func(t *testing.T) {
4393+
ds := NewDaemonset().WithContainer(corev1.Container{Name: "nvidia-driver-ctr"}).
4394+
WithInitContainer(corev1.Container{Name: "k8s-driver-manager"})
4395+
cpSpec := &gpuv1.ClusterPolicySpec{
4396+
Driver: gpuv1.DriverSpec{
4397+
Repository: "nvcr.io/nvidia",
4398+
Image: "driver",
4399+
ImagePullPolicy: "IfNotPresent",
4400+
Version: "580.126.16",
4401+
Manager: gpuv1.DriverManagerSpec{
4402+
Repository: "nvcr.io/nvidia/cloud-native",
4403+
Image: "k8s-driver-manager",
4404+
ImagePullPolicy: "IfNotPresent",
4405+
Version: "v0.8.0",
4406+
},
4407+
},
4408+
}
4409+
if tc.repoConfigEnabled {
4410+
cpSpec.Driver.RepoConfig = &gpuv1.DriverRepoConfigSpec{ConfigMapName: "test-repo-config"}
4411+
}
4412+
4413+
err := TransformDriver(ds.DaemonSet, cpSpec, ClusterPolicyController{
4414+
client: mockClient,
4415+
runtime: gpuv1.Containerd,
4416+
operatorNamespace: "test-ns",
4417+
logger: ctrl.Log.WithName("test"),
4418+
gpuNodeOSRelease: tc.osRelease,
4419+
gpuNodeOSTag: tc.osTag,
4420+
})
4421+
require.NoError(t, err)
4422+
4423+
driverContainer := findContainerByName(ds.Spec.Template.Spec.Containers, "nvidia-driver-ctr")
4424+
require.NotNil(t, driverContainer)
4425+
assertSubscriptionHostPathVolumesForTransform(t, ds.Spec.Template.Spec.Volumes, tc.expectedSubscriptionHostMap)
4426+
assert.Equal(t, tc.expectSubscriptionMounts, hasSubscriptionVolumeMountForTransform(driverContainer.VolumeMounts))
4427+
})
4428+
}
4429+
}
4430+
4431+
func hasSubscriptionVolumeMountForTransform(volumeMounts []corev1.VolumeMount) bool {
4432+
for _, volumeMount := range volumeMounts {
4433+
if strings.HasPrefix(volumeMount.Name, "subscription-config-") {
4434+
return true
4435+
}
4436+
}
4437+
return false
4438+
}
4439+
4440+
func assertSubscriptionHostPathVolumesForTransform(t *testing.T, volumes []corev1.Volume, expected map[string]corev1.HostPathType) {
4441+
t.Helper()
4442+
4443+
if expected == nil {
4444+
expected = map[string]corev1.HostPathType{}
4445+
}
4446+
4447+
actual := map[string]corev1.HostPathType{}
4448+
for _, volume := range volumes {
4449+
if !strings.HasPrefix(volume.Name, "subscription-config-") {
4450+
continue
4451+
}
4452+
require.NotNil(t, volume.HostPath)
4453+
require.NotNil(t, volume.HostPath.Type)
4454+
actual[volume.HostPath.Path] = *volume.HostPath.Type
4455+
}
4456+
4457+
assert.Equal(t, expected, actual)
4458+
}
4459+
43504460
// baseDriverDaemonSetSpec returns a minimal DaemonSetSpec representative of
43514461
// the post-transformation driver DaemonSet in the ClusterPolicy path.
43524462
// Only fields relevant to extractDriverInstallConfig extraction are

‎internal/state/driver_test.go‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package state
1818

1919
import (
2020
"bytes"
21+
"context"
2122
"os"
2223
"path/filepath"
2324
"strings"
@@ -34,8 +35,10 @@ import (
3435
apitypes "k8s.io/apimachinery/pkg/types"
3536
"k8s.io/client-go/kubernetes/scheme"
3637
"k8s.io/utils/ptr"
38+
"sigs.k8s.io/controller-runtime/pkg/client/fake"
3739

3840
nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
41+
"github.com/NVIDIA/gpu-operator/internal/consts"
3942
"github.com/NVIDIA/gpu-operator/internal/render"
4043
)
4144

@@ -44,6 +47,27 @@ const (
4447
manifestResultDir = "./testdata/golden"
4548
)
4649

50+
type testClusterInfo struct {
51+
runtime string
52+
openshiftVersion string
53+
}
54+
55+
func (i testClusterInfo) GetContainerRuntime() (string, error) {
56+
return i.runtime, nil
57+
}
58+
59+
func (i testClusterInfo) GetOpenshiftVersion() (string, error) {
60+
return i.openshiftVersion, nil
61+
}
62+
63+
func (i testClusterInfo) GetOpenshiftDriverToolkitImages() map[string]string {
64+
return nil
65+
}
66+
67+
func (i testClusterInfo) GetOpenshiftProxySpec() (*configv1.ProxySpec, error) {
68+
return nil, nil
69+
}
70+
4771
func getYAMLString(objs []*unstructured.Unstructured) (string, error) {
4872
s := json.NewSerializerWithOptions(json.DefaultMetaFactory, scheme.Scheme,
4973
scheme.Scheme, json.SerializerOptions{Yaml: true, Pretty: false, Strict: false})
@@ -60,6 +84,35 @@ func getYAMLString(objs []*unstructured.Unstructured) (string, error) {
6084
return sb.String(), nil
6185
}
6286

87+
func hasSubscriptionVolumeMount(volumeMounts []corev1.VolumeMount) bool {
88+
for _, volumeMount := range volumeMounts {
89+
if strings.HasPrefix(volumeMount.Name, "subscription-config-") {
90+
return true
91+
}
92+
}
93+
return false
94+
}
95+
96+
func assertSubscriptionHostPathVolumes(t *testing.T, volumes []corev1.Volume, expected map[string]corev1.HostPathType) {
97+
t.Helper()
98+
99+
if expected == nil {
100+
expected = map[string]corev1.HostPathType{}
101+
}
102+
103+
actual := map[string]corev1.HostPathType{}
104+
for _, volume := range volumes {
105+
if !strings.HasPrefix(volume.Name, "subscription-config-") {
106+
continue
107+
}
108+
require.NotNil(t, volume.HostPath)
109+
require.NotNil(t, volume.HostPath.Type)
110+
actual[volume.HostPath.Path] = *volume.HostPath.Type
111+
}
112+
113+
assert.Equal(t, expected, actual)
114+
}
115+
63116
func TestDriverRenderMinimal(t *testing.T) {
64117
// Construct a sample driver state manager
65118
const (
@@ -440,6 +493,69 @@ func TestDriverAdditionalConfigs(t *testing.T) {
440493
require.Equal(t, string(o), actual)
441494
}
442495

496+
func TestDriverAdditionalConfigsSubscriptionMounts(t *testing.T) {
497+
repoConfigMap := &corev1.ConfigMap{
498+
ObjectMeta: metav1.ObjectMeta{
499+
Name: "test-repo-config",
500+
Namespace: "test-ns",
501+
},
502+
Data: map[string]string{
503+
"redhat.repo": "[test-repo]",
504+
},
505+
}
506+
507+
testCases := []struct {
508+
description string
509+
osRelease string
510+
repoConfigEnabled bool
511+
expectSubscriptionMounts bool
512+
expectedSubscriptionHostMap map[string]corev1.HostPathType
513+
}{
514+
{
515+
description: "rhel with repo config skips host subscription mounts",
516+
osRelease: "rhel",
517+
repoConfigEnabled: true,
518+
expectSubscriptionMounts: false,
519+
},
520+
{
521+
description: "rhel without repo config mounts host subscription paths",
522+
osRelease: "rhel",
523+
expectSubscriptionMounts: true,
524+
expectedSubscriptionHostMap: map[string]corev1.HostPathType{
525+
"/etc/pki/entitlement": corev1.HostPathDirectory,
526+
"/etc/yum.repos.d/redhat.repo": corev1.HostPathFile,
527+
"/etc/rhsm": corev1.HostPathDirectory,
528+
},
529+
},
530+
}
531+
532+
for _, tc := range testCases {
533+
t.Run(tc.description, func(t *testing.T) {
534+
stateDriver := &stateDriver{
535+
stateSkel: stateSkel{
536+
client: fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(repoConfigMap).Build(),
537+
namespace: "test-ns",
538+
},
539+
}
540+
driver := &nvidiav1alpha1.NVIDIADriver{}
541+
if tc.repoConfigEnabled {
542+
driver.Spec.RepoConfig = &nvidiav1alpha1.DriverRepoConfigSpec{Name: "test-repo-config"}
543+
}
544+
545+
configs, err := stateDriver.getDriverAdditionalConfigs(
546+
context.Background(),
547+
driver,
548+
testClusterInfo{runtime: consts.Containerd},
549+
nodePool{osRelease: tc.osRelease, osVersion: tc.osRelease},
550+
)
551+
require.NoError(t, err)
552+
553+
assertSubscriptionHostPathVolumes(t, configs.Volumes, tc.expectedSubscriptionHostMap)
554+
assert.Equal(t, tc.expectSubscriptionMounts, hasSubscriptionVolumeMount(configs.VolumeMounts))
555+
})
556+
}
557+
}
558+
443559
func TestDriverOpenshiftDriverToolkit(t *testing.T) {
444560
const (
445561
testName = "driver-openshift-drivertoolkit"

‎internal/state/driver_volumes.go‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -182,10 +182,19 @@ func (s *stateDriver) getDriverAdditionalConfigs(ctx context.Context, cr *v1alph
182182

183183
// set up subscription entitlements for RHEL(using K8s with a non-CRIO runtime) and SLES
184184
if (pool.osRelease == "rhel" && openshiftVersion == "" && runtime != consts.CRIO) || pool.osRelease == "sles" || pool.osRelease == "sl-micro" {
185-
logger.Info("Mounting subscriptions into the driver container", "OS", pool.osVersion)
186-
pathToVolumeSource, err := getSubscriptionPathsToVolumeSources(pool.osRelease)
187-
if err != nil {
188-
return nil, fmt.Errorf("ERROR: failed to get path items for subscription entitlements: %v", err)
185+
pathToVolumeSource := MountPathToVolumeSource{}
186+
// Custom repo ConfigMap supplies yum repos in offline/air-gapped installs. Skip
187+
// mounting host RHSM paths (/etc/pki/entitlement, redhat.repo, /etc/rhsm): they may
188+
// be missing or not directories on minimal nodes, and are not needed when packages
189+
// come only from the mounted repo ConfigMap.
190+
if cr.Spec.IsRepoConfigEnabled() && pool.osRelease == "rhel" {
191+
logger.Info("Skipping host subscription mounts because repoConfig is enabled", "OS", pool.osVersion)
192+
} else {
193+
logger.Info("Mounting subscriptions into the driver container", "OS", pool.osVersion)
194+
pathToVolumeSource, err = getSubscriptionPathsToVolumeSources(pool.osRelease)
195+
if err != nil {
196+
return nil, fmt.Errorf("ERROR: failed to get path items for subscription entitlements: %w", err)
197+
}
189198
}
190199

191200
// sort host path volumes to ensure ordering is preserved when adding to pod spec

0 commit comments

Comments
 (0)