Switch gpu-operator image to distroless base image

Signed-off-by: Christopher Desiniotis <cdesiniotis@nvidia.com>

Author: cdesiniotis

.common-ci.yml

@@ -82,11 +82,6 @@ trigger-pipeline:
     -  '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes'
 
 # Define targets for the gpu-operator image
-.dist-ubi9:
-  variables:
-    DIST: ubi9
-    CVE_UPDATES: "cyrus-sasl-lib"
-
 .target-gpu-operator:
   variables:
     IMAGE_NAME: "${CI_REGISTRY_IMAGE}"
@@ -122,7 +117,7 @@ trigger-pipeline:
 
     # Since OUT_IMAGE_NAME and OUT_IMAGE_VERSION are set, this will push the CI image to the
     # Target
-    - make push-${DIST}
+    - make push-image
 
 .release-bundle:
   stage: release
@@ -174,15 +169,13 @@ trigger-pipeline:
 release:staging-gpu-operator:
   extends:
     - .release:staging
-    - .dist-ubi9
     - .target-gpu-operator
   variables:
     OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/staging/gpu-operator"
 
 release:staging-latest-gpu-operator:
   extends:
     - .release:staging
-    - .dist-ubi9
     - .target-gpu-operator
   variables:
     OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/staging/gpu-operator"

.github/workflows/ci.yaml

@@ -133,9 +133,6 @@ jobs:
   build-gpu-operator-arm64:
     needs: [go-check, go-test, go-build]
     runs-on: ubuntu-24.04-arm
-    strategy:
-      matrix:
-        dist: [ubi9]
     steps:
       - uses: actions/checkout@v4
         name: Check out code
@@ -172,13 +169,10 @@ jobs:
           VERSION: ${COMMIT_SHORT_SHA}-arm64
         run: |
           echo "${VERSION}"
-          make build-${{ matrix.dist }}
+          make build-image
   build-gpu-operator-amd64:
     needs: [go-check, go-test, go-build]
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        dist: [ubi9]
     steps:
       - uses: actions/checkout@v4
         name: Check out code
@@ -215,14 +209,11 @@ jobs:
           VERSION: ${COMMIT_SHORT_SHA}-amd64
         run: |
           echo "${VERSION}"
-          make build-${{ matrix.dist }}
+          make build-image
 
   build-multi-arch-images:
     needs: [build-gpu-operator-arm64, build-gpu-operator-amd64]
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        dist: [ubi9]
     steps:
       - uses: actions/checkout@v4
         name: Check out code

.gitlab-ci.yml

@@ -90,12 +90,11 @@ unit-tests:
     - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
     - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
   script:
-    - make build-${DIST}
+    - make build-image
 
 build:gpu-operator:
   extends:
     - .image-build
-    - .dist-ubi9
     - .target-gpu-operator
 
 .e2e_defaults:
@@ -108,8 +107,6 @@ build:gpu-operator:
     OPERATOR_VERSION: "${CI_COMMIT_SHORT_SHA}"
     OPERATOR_IMAGE: "${CI_REGISTRY_IMAGE}"
     GPU_PRODUCT_NAME: "Tesla-T4"
-  extends:
-    - .dist-ubi9
   except:
     variables:
       - $CI_COMMIT_MESSAGE =~ /skip-end-to-end-tests/

.nvidia-ci.yml

@@ -50,12 +50,11 @@ variables:
       regctl manifest get ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION} --list > /dev/null && echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}" || ( echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION} does not exist" && sleep infinity )
   script:
     - regctl registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}"
-    - make IMAGE=${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION} OUT_IMAGE=${OUT_IMAGE_NAME}:${CI_COMMIT_SHORT_SHA} push-${DIST}
+    - make IMAGE=${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION} OUT_IMAGE=${OUT_IMAGE_NAME}:${CI_COMMIT_SHORT_SHA} push-image
 
 image:gpu-operator:
   extends:
     - .image-pull
-    - .dist-ubi9
     - .target-gpu-operator
 
 # We skip the integration tests for the internal CI:
@@ -101,7 +100,6 @@ image:gpu-operator:
 .scan:gpu-operator:
   extends:
     - .scan
-    - .dist-ubi9
     - .target-gpu-operator
   needs:
     - image:gpu-operator
@@ -130,7 +128,6 @@ scan:gpu-operator-arm64:
 release:ngc-gpu-operator:
   extends:
     - .release:ngc
-    - .dist-ubi9
     - .target-gpu-operator
 
 # Define the external image signing steps for NGC

Makefile

@@ -71,8 +71,6 @@ else
 GOBIN=$(shell go env GOBIN)
 endif
 
-all: gpu-operator
-
 GOOS ?= linux
 VERSION_PKG = github.com/NVIDIA/gpu-operator/internal/info
 
@@ -258,43 +256,36 @@ cov-report: coverage install-tools
 	$(GCOV2LCOV) -infile $(COVERAGE_FILE) -outfile lcov.info
 
 ##### Public rules #####
-DISTRIBUTIONS := ubi9
-DEFAULT_PUSH_TARGET := ubi9
-
-PUSH_TARGETS := $(patsubst %,push-%, $(DISTRIBUTIONS))
-BUILD_TARGETS := $(patsubst %,build-%, $(DISTRIBUTIONS))
-TEST_TARGETS := $(patsubst %,test-%, $(DISTRIBUTIONS))
+PUSH_TARGETS := push-image
+BUILD_TARGETS := build-image
+TEST_TARGETS := test
 
 ifneq ($(BUILD_MULTI_ARCH_IMAGES),true)
 include $(CURDIR)/native-only.mk
 else
 include $(CURDIR)/multi-arch.mk
 endif
 
-ALL_TARGETS := $(DISTRIBUTIONS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS) docker-image
+ALL_TARGETS := $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS) docker-image
 .PHONY: $(ALL_TARGETS)
 
 build-%: DOCKERFILE = $(CURDIR)/docker/Dockerfile
 
-$(DISTRIBUTIONS): %: build-%
-$(BUILD_TARGETS): build-%:
+build-image:
 	DOCKER_BUILDKIT=1 \
 		$(DOCKER) $(BUILDX) build --pull \
 		$(DOCKER_BUILD_OPTIONS) \
 		$(DOCKER_BUILD_PLATFORM_OPTIONS) \
 		--tag $(IMAGE) \
 		--build-arg VERSION="$(VERSION)" \
 		--build-arg BUILDER_IMAGE="$(BUILDER_IMAGE)" \
-		--build-arg CUDA_SAMPLE_IMAGE=nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda$(CUDA_SAMPLES_VERSION) \
 		--build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
-		--build-arg CVE_UPDATES="$(CVE_UPDATES)" \
 		--build-arg GIT_COMMIT="$(GIT_COMMIT)" \
 		--file $(DOCKERFILE) $(CURDIR)
 
 # Provide a utility target to build the images to allow for use in external tools.
 # This includes https://github.com/openshift-psap/ci-artifacts
 docker-image: OUT_IMAGE ?= $(IMAGE_NAME):$(IMAGE_TAG)
-docker-image: ${DEFAULT_PUSH_TARGET}
 
 install-tools:
 	@echo Installing tools from tools.go

cmd/nvidia-validator/main.go

@@ -217,6 +217,8 @@ const (
 	appComponentLabelKey = "app.kubernetes.io/component"
 	// wslNvidiaSMIPath indicates the path to the nvidia-smi binary on WSL
 	wslNvidiaSMIPath = "/usr/lib/wsl/lib/nvidia-smi"
+	// shell indicates what shell to use when invoking commands in a subprocess
+	shell = "sh"
 )
 
 func main() {
@@ -616,6 +618,7 @@ func runCommandWithWait(command string, args []string, sleepSeconds int, silent
 		fmt.Printf("running command %s with args %v\n", command, args)
 		err := cmd.Run()
 		if err != nil {
+			log.Warningf("error running command: %v", err)
 			fmt.Printf("command failed, retrying after %d seconds\n", sleepSeconds)
 			time.Sleep(time.Duration(sleepSeconds) * time.Second)
 			continue
@@ -649,7 +652,7 @@ func setEnvVar(envvars []string, key, value string) []string {
 // For driver container installs, check existence of .driver-ctr-ready to confirm running driver
 // container has completed and is in Ready state.
 func assertDriverContainerReady(silent bool) error {
-	command := "bash"
+	command := shell
 	args := []string{"-c", "stat /run/nvidia/validations/.driver-ctr-ready"}
 
 	if withWaitFlag {
@@ -932,7 +935,7 @@ func (n *NvidiaFs) validate() error {
 
 func (n *NvidiaFs) runValidation(silent bool) error {
 	// check for nvidia_fs module to be loaded
-	command := "bash"
+	command := shell
 	args := []string{"-c", "lsmod | grep nvidia_fs"}
 
 	if withWaitFlag {
@@ -1067,7 +1070,7 @@ func (m *MOFED) validate() error {
 
 func (m *MOFED) runValidation(silent bool) error {
 	// check for mlx5_core module to be loaded
-	command := "bash"
+	command := shell
 	args := []string{"-c", "lsmod | grep mlx5_core"}
 
 	// If MOFED container is running then use readiness flag set by the driver container instead
@@ -1632,7 +1635,7 @@ func (c *CCManager) setKubeClient(kubeClient kubernetes.Interface) {
 
 // Check that the ccManager container is ready after applying required ccMode
 func assertCCManagerContainerReady(silent, withWaitFlag bool) error {
-	command := "bash"
+	command := shell
 	args := []string{"-c", "stat /run/nvidia/validations/.cc-manager-ctr-ready"}
 
 	if withWaitFlag {

docker/Dockerfile

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG CUDA_SAMPLE_IMAGE=undefined
 ARG GOLANG_VERSION=x.x.x
 
 FROM nvcr.io/nvidia/cuda:12.9.0-base-ubi9 as builder
@@ -56,17 +55,17 @@ ARG VERSION="unknown"
 ARG GIT_COMMIT="unknown"
 RUN make cmds
 
-FROM ${CUDA_SAMPLE_IMAGE} AS sample-builder
-
-FROM nvcr.io/nvidia/cuda:12.9.0-base-ubi9
+# Install must-gather dependency: `kubectl`
+ARG TARGETARCH
+RUN OS_ARCH=${TARGETARCH/x86_64/amd64} && OS_ARCH=${OS_ARCH/aarch64/arm64} && curl -LO https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/${OS_ARCH}/kubectl
+RUN chmod +x ./kubectl
+RUN mv ./kubectl /usr/local/bin
 
-# Remove CUDA libs(compat etc) in favor of libs installed by the NVIDIA driver
-RUN dnf remove -y cuda-*
+FROM nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda12.5.0 AS sample-builder
 
-RUN dnf install -y \
-        kmod \
-        pciutils && \
-    rm -rf /var/cache/yum/*
+# The C/C++ distroless image is used as a base since the CUDA vectorAdd
+# sample application depends on C/C++ libraries.
+FROM nvcr.io/nvidia/distroless/cc:v3.1.7-dev
 
 ENV NVIDIA_VISIBLE_DEVICES=void
 
@@ -84,40 +83,27 @@ LABEL vsc-ref=${GIT_COMMIT}
 
 WORKDIR /
 COPY --from=builder /workspace/gpu-operator /usr/bin/
+COPY --from=builder /workspace/kubectl /usr/bin/
 COPY --from=builder /workspace/nvidia-validator /usr/bin/
 COPY --from=sample-builder /cuda-samples/vectorAdd /usr/bin/vectorAdd
+# TODO: Copy the compat libs from the 'sample-builder' image instead.
+# The current 'sample-builder' image does not contain the compat libs in the ARM variant.
+# Once new sample images are published that contain the compat libs, we can update the below.
+COPY --from=builder /usr/local/cuda/compat /usr/local/cuda/compat
 
-# gpu-operator manifests
-RUN mkdir -p /opt/gpu-operator/manifests
 COPY assets /opt/gpu-operator/
 COPY manifests /opt/gpu-operator/manifests
+COPY validator/manifests /opt/validator/manifests
 
-# validator manifests
-RUN mkdir -p /opt/validator/manifests
-COPY validator/manifests/plugin-workload-validation.yaml /opt/validator/manifests
-COPY validator/manifests/cuda-workload-validation.yaml /opt/validator/manifests
-
-RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
 COPY hack/must-gather.sh /usr/bin/gather
 
-# Install must-gather dependency: `kubectl`
-ARG TARGETARCH
-RUN OS_ARCH=${TARGETARCH/x86_64/amd64} && OS_ARCH=${OS_ARCH/aarch64/arm64} && curl -LO https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/${OS_ARCH}/kubectl
-RUN chmod +x ./kubectl
-RUN mv ./kubectl /usr/local/bin
-
 # Add CRD resource into the image for helm upgrades
 COPY deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml /opt/gpu-operator/nvidia.com_clusterpolicies.yaml
 COPY deployments/gpu-operator/crds/nvidia.com_nvidiadrivers.yaml /opt/gpu-operator/nvidia.com_nvidiadrivers.yaml
 COPY deployments/gpu-operator/charts/node-feature-discovery/crds/nfd-api-crds.yaml /opt/gpu-operator/nfd-api-crds.yaml
 
-# Install / upgrade packages here that are required to resolve CVEs
-ARG CVE_UPDATES
-RUN if [ -n "${CVE_UPDATES}" ]; then \
-        dnf update -y ${CVE_UPDATES} && \
-        rm -rf /var/cache/yum/*; \
-    fi
-
 USER 65532:65532
 
+COPY LICENSE /licenses/
+
 ENTRYPOINT ["/usr/bin/gpu-operator"]

multi-arch.mk

@@ -18,7 +18,7 @@ DOCKER_BUILD_OPTIONS = --output=type=image,push=$(PUSH_ON_BUILD) --provenance=$(
 DOCKER_BUILD_PLATFORM_OPTIONS ?= --platform=linux/amd64,linux/arm64
 
 REGCTL ?= regctl
-$(PUSH_TARGETS): push-%:
+push-image:
 	$(REGCTL) \
 	        image copy \
 	        $(IMAGE) $(OUT_IMAGE)

native-only.mk

@@ -15,7 +15,8 @@
 PUSH_ON_BUILD ?= false
 DOCKER_BUILD_PLATFORM_OPTIONS ?= --platform=linux/amd64
 DOCKER_BUILD_OPTIONS = --output=type=image,push=$(PUSH_ON_BUILD) --provenance=$(ATTACH_ATTESTATIONS) --sbom=$(ATTACH_ATTESTATIONS)
-$(PUSH_TARGETS): OUT_IMAGE ?= $(IMAGE_NAME):$(IMAGE_TAG)
-$(PUSH_TARGETS): push-%:
+
+push-image: OUT_IMAGE ?= $(IMAGE_NAME):$(IMAGE_TAG)
+push-image:
 	$(DOCKER) tag "$(IMAGE_NAME):$(VERSION)-$(DEFAULT_PUSH_TARGET)" "$(OUT_IMAGE)"
 	$(DOCKER) push "$(OUT_IMAGE)"

versions.mk

@@ -23,6 +23,4 @@ GOLANG_VERSION ?= 1.24.4
 
 GOLANGCI_LINT_VERSION ?= v2.1.6
 
-CUDA_SAMPLES_VERSION ?= 12.5.0
-
 GIT_COMMIT ?= $(shell git describe --match="" --dirty --long --always 2> /dev/null || echo "")