From 8d919d2d02ce28adb6af59cfd85297375ef1a863 Mon Sep 17 00:00:00 2001 From: Evgeny Uglov Date: Tue, 27 Jan 2026 12:07:21 -0500 Subject: [PATCH] Mirror internal repository with cleaned references --- .gitignore | 6 + Dockerfile | 22 +- Makefile | 125 +- README.md | 18 +- cmd/csi-powerstore/main.go | 158 +- cmd/csi-powerstore/main_test.go | 230 +- core/semver/semver.go | 2 +- dell-csi-helm-installer/csi-install.sh | 12 +- dell-csi-helm-installer/csi-uninstall.sh | 14 + dell-csi-helm-installer/verify.sh | 21 + docker.mk | 66 - go.mod | 160 +- go.sum | 380 +-- helper.mk | 17 + images.mk | 22 + mkdocs.yml | 7 + mocks/FcConnector.go | 18 + mocks/NodeInterface.go | 2 +- mocks/NodeLabelsModifier.go | 66 - mocks/NodeLabelsRetriever.go | 178 -- mocks/NodeVolumePublisher.go | 11 +- mocks/NodeVolumeStager.go | 11 +- mocks/VolumeStager.go | 13 +- overrides.mk | 29 + pkg/array/array.go | 172 +- pkg/array/array_test.go | 480 ++- pkg/array/metro_utils.go | 200 ++ pkg/array/metro_utils_test.go | 602 ++++ pkg/controller/base.go | 2 +- pkg/controller/controller.go | 796 +++-- .../controller_node_to_array_connectivity.go | 7 +- pkg/controller/controller_test.go | 2000 ++++++++++++- pkg/controller/creator.go | 11 +- pkg/controller/creator_test.go | 2 +- pkg/controller/csi_extension_server.go | 49 +- pkg/controller/csi_extension_server_test.go | 47 + pkg/controller/publisher.go | 36 +- pkg/controller/replication.go | 327 +- pkg/controller/replication_test.go | 185 +- pkg/helpers/utils.go | 5 + pkg/helpers/utils_test.go | 3 +- pkg/identifiers/envvars.go | 19 +- pkg/identifiers/fs/fs.go | 16 +- pkg/identifiers/identifiers.go | 162 +- pkg/identifiers/identifiers_test.go | 222 +- pkg/identifiers/k8sutils/k8sutils.go | 221 +- pkg/identifiers/k8sutils/k8sutils_test.go | 546 +++- pkg/identifiers/logger.go | 16 +- pkg/identity/identity_test.go | 2 +- pkg/interceptors/interceptors.go | 13 +- pkg/interceptors/interceptors_test.go | 4 +- pkg/monitor/event.go | 274 ++ pkg/monitor/event_test.go | 778 +++++ pkg/node/acl.go | 5 +- pkg/node/base.go | 34 +- pkg/node/ephemeral.go | 7 +- pkg/node/node.go | 727 ++++- pkg/node/node_connectivity_checker.go | 46 +- pkg/node/node_connectivity_checker_test.go | 74 +- pkg/node/node_test.go | 2622 +++++++++++++---- pkg/node/publisher.go | 42 +- pkg/node/stager.go | 137 +- pkg/node/stager_test.go | 167 +- pkg/provider/provider.go | 83 - pkg/provider/provider_test.go | 99 - pkg/service/controller.go | 224 -- pkg/service/controller_test.go | 584 ---- pkg/service/identity.go | 42 - pkg/service/identity_test.go | 42 - pkg/service/node.go | 203 -- pkg/service/node_test.go | 367 --- pkg/service/service.go | 196 -- pkg/service/service_test.go | 134 - samples/secret/secret.yaml | 146 +- .../powerstore-nfs-replication.yaml | 100 + tests/e2e/k8s/README.md | 22 - tests/e2e/k8s/e2e-values.yaml | 34 - tests/e2e/k8s/externalAccess.go | 303 -- tests/e2e/k8s/go.mod | 212 -- tests/e2e/k8s/go.sum | 700 ----- tests/e2e/k8s/run.sh | 23 - tests/e2e/k8s/suite_test.go | 76 - .../statefulset/statefulset.yaml | 47 - tests/e2e/k8s/utils.go | 346 --- tests/sanity/README.md | 9 +- .../sanity/setup-driver-controller-sanity.sh | 4 +- tests/sanity/setup-driver-node-sanity.sh | 4 +- 87 files changed, 10379 insertions(+), 6267 deletions(-) delete mode 100644 docker.mk create mode 100644 helper.mk create mode 100644 images.mk create mode 100644 mkdocs.yml delete mode 100644 mocks/NodeLabelsModifier.go delete mode 100644 mocks/NodeLabelsRetriever.go create mode 100644 overrides.mk create mode 100644 pkg/array/metro_utils.go create mode 100644 pkg/array/metro_utils_test.go create mode 100644 pkg/monitor/event.go create mode 100644 pkg/monitor/event_test.go delete mode 100644 pkg/provider/provider.go delete mode 100644 pkg/provider/provider_test.go delete mode 100644 pkg/service/controller.go delete mode 100644 pkg/service/controller_test.go delete mode 100644 pkg/service/identity.go delete mode 100644 pkg/service/identity_test.go delete mode 100644 pkg/service/node.go delete mode 100644 pkg/service/node_test.go delete mode 100644 pkg/service/service.go delete mode 100644 pkg/service/service_test.go create mode 100644 samples/storageclass/powerstore-nfs-replication.yaml delete mode 100644 tests/e2e/k8s/README.md delete mode 100644 tests/e2e/k8s/e2e-values.yaml delete mode 100644 tests/e2e/k8s/externalAccess.go delete mode 100644 tests/e2e/k8s/go.mod delete mode 100644 tests/e2e/k8s/go.sum delete mode 100755 tests/e2e/k8s/run.sh delete mode 100644 tests/e2e/k8s/suite_test.go delete mode 100644 tests/e2e/k8s/testing-manifests/statefulset/statefulset.yaml delete mode 100644 tests/e2e/k8s/utils.go diff --git a/.gitignore b/.gitignore index 29717bea..76c6a42f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,11 +3,15 @@ semver.mk vars.mk Dockerfile-debug +go-code-tester* +csm-common.mk +*coverage*.txt # test directories and artifacts service/c.out service/test/ vendor +**/test-data* # IDE .idea @@ -28,3 +32,5 @@ gosecresults.csv # logs *.log +vendor/ + diff --git a/Dockerfile b/Dockerfile index c1b62fa6..c3b39be1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,33 +13,33 @@ # some arguments that must be supplied ARG GOIMAGE ARG BASEIMAGE +ARG VERSION="2.16.0" # Stage to build the driver FROM $GOIMAGE as builder +ARG VERSION -WORKDIR /workspace -COPY . . +RUN mkdir -p /go/src/csi-powerstore +COPY ./ /go/src/csi-powerstore -RUN go generate ./cmd/csi-powerstore -RUN GOOS=linux CGO_ENABLED=0 go build -o csi-powerstore ./cmd/csi-powerstore +WORKDIR /go/src/csi-powerstore +RUN make build IMAGE_VERSION=$VERSION && \ + rm -rf /go/src/csi-powerstore/vendor # Stage to build the driver image FROM $BASEIMAGE +ARG VERSION WORKDIR / LABEL vendor="Dell Technologies" \ maintainer="Dell Technologies" \ name="csi-powerstore" \ summary="CSI Driver for Dell EMC PowerStore" \ description="CSI Driver for provisioning persistent storage from Dell EMC PowerStore" \ - release="1.15.0" \ - version="2.15.0" \ + release="1.16.0" \ + version=$VERSION \ license="Apache-2.0" COPY licenses /licenses -# validate some cli utilities are found -RUN which mkfs.ext4 -RUN which mkfs.xfs -RUN echo "export PATH=$PATH:/sbin:/bin" > /etc/profile.d/ubuntu_path.sh +COPY --from=builder /go/src/csi-powerstore/csi-powerstore /csi-powerstore -COPY --from=builder /workspace/csi-powerstore / ENTRYPOINT ["/csi-powerstore"] diff --git a/Makefile b/Makefile index 1dd98704..abae4a27 100644 --- a/Makefile +++ b/Makefile @@ -1,76 +1,45 @@ +# Copyright © 2026 Dell Inc. or its subsidiaries. All Rights Reserved. # -# -# Copyright © 2020-2025 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -# for variables override --include vars.mk - -all: clean build - -# Tag parameters -ifndef TAGMSG - TAGMSG="CSI Spec 1.6" -endif +# Dell Technologies, Dell and other trademarks are trademarks of Dell Inc. +# or its subsidiaries. Other trademarks may be trademarks of their respective +# owners. + +include images.mk + +all: help + +# This will be overridden during image build. +IMAGE_VERSION ?= 0.0.0 +LDFLAGS = "-X main.ManifestSemver=$(IMAGE_VERSION)" + +# Help target, prints usefule information +help: + @echo + @echo "The following targets are commonly used:" + @echo + @echo "build - Builds the code locally" + @echo "check - Runs the suite of code checking tools: lint, format, etc" + @echo "clean - Cleans the local build" + @echo "images - Builds the code within a golang container and then creates the driver image" + @echo "push - Pushes the built container to a target registry" + @echo "unit-test - Runs the unit tests" + @echo "vendor - Downloads a vendor list (local copy) of repositories required to compile the repo." clean: - rm -f core/core_generated.go - rm -f semver.mk + rm -f semver.mk core/core_generated.go + rm -rf vendor + rm -f csi-powerstore go clean -cache -generate: - go generate ./cmd/csi-powerstore - -build: generate - GOOS=linux CGO_ENABLED=0 go build ./cmd/csi-powerstore - -install: generate - GOOS=linux CGO_ENABLED=0 go install ./cmd/csi-powerstore - -# Tags the release with the Tag parameters set above -tag: - go run core/semver/semver.go -f mk >semver.mk - make -f docker.mk tag TAGMSG='$(TAGMSG)' - -# Generates the docker container (but does not push) -docker: - go run core/semver/semver.go -f mk >semver.mk - make -f docker.mk docker - -# Same as `docker` but without cached layers and will pull latest version of base image -docker-no-cache: - go run core/semver/semver.go -f mk >semver.mk - make -f docker.mk docker-no-cache - -# Pushes container to the repository -push: docker - make -f docker.mk push - -check: gosec - gofmt -w ./. -ifeq (, $(shell which golint)) - go install golang.org/x/lint/golint@latest -endif - golint -set_exit_status ./. - go vet ./... +build: generate vendor + GOOS=linux CGO_ENABLED=0 go build -mod=vendor -ldflags $(LDFLAGS) ./cmd/csi-powerstore mocks: mockery unit-test: go-code-tester GITHUB_OUTPUT=/dev/null \ - ./go-code-tester 90 "." "" "true" "" "" "./mocks|./v2/core|./tests" + ./go-code-tester 90 "." "" "true" "" "" "./mocks|./v2/core|./tests|./replace" test: cd ./pkg; go test -race -cover -coverprofile=coverage.out ./... @@ -78,32 +47,8 @@ test: coverage: cd ./pkg; go tool cover -html=coverage.out -o coverage.html -gosec: -ifeq (, $(shell which gosec)) - go install github.com/securego/gosec/v2/cmd/gosec@latest - $(shell $(GOBIN)/gosec -quiet -log gosec.log -out=gosecresults.csv -fmt=csv ./...) -else - $(shell gosec -quiet -log gosec.log -out=gosecresults.csv -fmt=csv ./...) -endif - @echo "Logs are stored at gosec.log, Outputfile at gosecresults.csv" - go-code-tester: - curl -o go-code-tester -L https://raw.githubusercontent.com/dell/common-github-actions/main/go-code-tester/entrypoint.sh \ - && chmod +x go-code-tester - -.PHONY: actions action-help -actions: ## Run all GitHub Action checks that run on a pull request creation - @echo "Running all GitHub Action checks for pull request events..." - @act -l | grep -v ^Stage | grep pull_request | grep -v image_security_scan | awk '{print $$2}' | while read WF; do \ - echo "Running workflow: $${WF}"; \ - act pull_request --no-cache-server --platform ubuntu-latest=ghcr.io/catthehacker/ubuntu:act-latest --job "$${WF}"; \ - done - -action-help: ## Echo instructions to run one specific workflow locally - @echo "GitHub Workflows can be run locally with the following command:" - @echo "act pull_request --no-cache-server --platform ubuntu-latest=ghcr.io/catthehacker/ubuntu:act-latest --job " - @echo "" - @echo "Where '' is a Job ID returned by the command:" - @echo "act -l" - @echo "" - @echo "NOTE: if act is not installed, it can be downloaded from https://github.com/nektos/act" + git clone --depth 1 git@github.com:CSM/actions.git temp-repo + cp temp-repo/go-code-tester/entrypoint.sh ./go-code-tester + chmod +x go-code-tester + rm -rf temp-repo diff --git a/README.md b/README.md index 2d54a801..2129adbd 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,20 @@ +# :lock: **Important Notice** +Starting with the release of **Container Storage Modules v1.16.0**, this repository will no longer be maintained as an open source project. Future development will continue under a closed source model. This change reflects our commitment to delivering even greater value to our customers by enabling faster innovation and more deeply integrated features with the Dell storage portfolio.
+For existing customers using Dell’s Container Storage Modules, you will continue to receive: +* **Ongoing Support & Community Engagement**
+ You will continue to receive high-quality support through Dell Support and our community channels. Your experience of engaging with the Dell community remains unchanged. +* **Streamlined Deployment & Updates**
+ Deployment and update processes will remain consistent, ensuring a smooth and familiar experience. +* **Access to Documentation & Resources**
+ All documentation and related materials will remain publicly accessible, providing transparency and technical guidance. +* **Continued Access to Current Open Source Version**
+ The current open-source version will remain available under its existing license for those who rely on it. + +Moving to a closed source model allows Dell’s development team to accelerate feature delivery and enhance integration across our Enterprise Kubernetes Storage solutions ultimately providing a more seamless and robust experience.
+We deeply appreciate the contributions of the open source community and remain committed to supporting our customers through this transition.
+ +For questions or access requests, please contact the maintainers via [Dell Support](https://www.dell.com/support/kbdoc/en-in/000188046/container-storage-interface-csi-drivers-and-container-storage-modules-csm-how-to-get-support). + # CSI Driver for Dell PowerStore [![Go Report Card](https://goreportcard.com/badge/github.com/dell/csi-powerstore?style=flat-square)](https://goreportcard.com/report/github.com/dell/csi-powerstore) @@ -55,4 +72,3 @@ If you want to use NVMe/FC be sure that the NVMeFC zoning of the Host Bus Adapte ## Documentation For more detailed information on the driver, please refer to [Container Storage Modules documentation](https://dell.github.io/csm-docs/). - diff --git a/cmd/csi-powerstore/main.go b/cmd/csi-powerstore/main.go index b3b211ec..cff34461 100644 --- a/cmd/csi-powerstore/main.go +++ b/cmd/csi-powerstore/main.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,30 +20,35 @@ import ( "context" "fmt" "os" + "strconv" "strings" "time" - "github.com/dell/csi-powerstore/v2/core" + "github.com/dell/csi-powerstore/v2/pkg/array" "github.com/dell/csi-powerstore/v2/pkg/controller" "github.com/dell/csi-powerstore/v2/pkg/identifiers" "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" "github.com/dell/csi-powerstore/v2/pkg/identity" "github.com/dell/csi-powerstore/v2/pkg/interceptors" + "github.com/dell/csi-powerstore/v2/pkg/monitor" "github.com/dell/csi-powerstore/v2/pkg/node" - "github.com/dell/csi-powerstore/v2/pkg/provider" "github.com/dell/csi-powerstore/v2/pkg/tracer" + drController "github.com/dell/csm-dr/pkg/controller" + "github.com/dell/csmlog" "github.com/dell/gocsi" csictx "github.com/dell/gocsi/context" "github.com/dell/gofsutil" "github.com/fsnotify/fsnotify" grpc_opentracing "github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing" "github.com/opentracing/opentracing-go" - log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus" "github.com/spf13/viper" "github.com/uber/jaeger-client-go/config" "google.golang.org/grpc" ) +var log = csmlog.GetLogger() + //go:generate go generate ../../core func init() { @@ -58,7 +63,7 @@ func init() { initilizeDriverConfigParams() // If we don't set this env gocsi will overwrite log level with default Info level - _ = os.Setenv(gocsi.EnvVarLogLevel, log.GetLevel().String()) + _ = os.Setenv(gocsi.EnvVarLogLevel, csmlog.GetLevel().String()) } func updateDriverName() { @@ -68,9 +73,10 @@ func updateDriverName() { } func initilizeDriverConfigParams() { + log.SetLevel(csmlog.InfoLevel) paramsPath, ok := csictx.LookupEnv(context.Background(), identifiers.EnvConfigParamsFilePath) if !ok { - log.Warnf("config path X_CSI_POWERSTORE_CONFIG_PARAMS_PATH is not specified") + log.Warn("config path X_CSI_POWERSTORE_CONFIG_PARAMS_PATH is not specified") } paramsViper := viper.New() @@ -80,7 +86,7 @@ func initilizeDriverConfigParams() { err := paramsViper.ReadInConfig() // if unable to read configuration file, default values will be used in updateDriverConfigParams if err != nil { - log.WithError(err).Error("unable to read config file, using default values") + log.Warnf("unable to read config file, using default values %s ", err.Error()) } paramsViper.WatchConfig() paramsViper.OnConfigChange(func(e fsnotify.Event) { @@ -91,12 +97,21 @@ func initilizeDriverConfigParams() { updateDriverConfigParams(paramsViper) } +var ManifestSemver string + func main() { + log.SetLevel(csmlog.InfoLevel) f := &fs.Fs{Util: &gofsutil.FS{}} identifiers.RmSockFile(f) - identityService := identity.NewIdentityService(identifiers.Name, core.SemVer, identifiers.Manifest) + if ManifestSemver != "" { + log.Info("ManifestVersion isn't empty, setting it") + identifiers.ManifestSemver = ManifestSemver + identifiers.Manifest["semver"] = ManifestSemver + } + + identityService := identity.NewIdentityService(identifiers.Name, ManifestSemver, identifiers.Manifest) var controllerService *controller.Service var nodeService *node.Service @@ -111,44 +126,51 @@ func main() { identifiers.Name = name } identifiers.SetAPIPort(context.Background()) - if strings.EqualFold(mode, "controller") { - cs := &controller.Service{ - Fs: f, - } - err := cs.UpdateArrays(configPath, f) - if err != nil { - log.Fatalf("couldn't initialize arrays in controller service: %s", err.Error()) - } + var nodeName string + var arrayLocker *array.Locker + + isCSMDREnabled, err := strconv.ParseBool(os.Getenv(identifiers.EnvCSMDREnabled)) + if err != nil { + log.Infof("Error parsing %s: %s. Defaulting to true", identifiers.EnvCSMDREnabled, err.Error()) + isCSMDREnabled = true + } - err = cs.Init() + if strings.EqualFold(mode, "controller") { + + var err error + controllerService, err = initControllerService(f, configPath) if err != nil { - log.Fatalf("couldn't create controller service: %s", err.Error()) + log.Fatalf("couldn't initialize controller service: %s", err.Error()) } - controllerService = cs + arrayLocker = &controllerService.Locker + controllerService.IsCSMDREnabled = isCSMDREnabled } else if strings.EqualFold(mode, "node") { - ns := &node.Service{ - Fs: f, - } - - err := ns.UpdateArrays(configPath, f) + var err error + nodeService, err = initNodeService(f, configPath) if err != nil { - log.Fatalf("couldn't initialize arrays in node service: %s", err.Error()) + log.Fatalf("couldn't initialize node service: %s", err.Error()) } - err = ns.Init() + nodeName = os.Getenv(identifiers.EnvKubeNodeName) + arrayLocker = &nodeService.Locker + } + + if isCSMDREnabled { + // Initialize CSM DR volume journal reconciler. + log.Infof("Initializing CSM-DR controller ") + _, err := drController.Initialize(nodeService, controllerService, arrayLocker, mode, nodeName, ":8080", false, ":8081") if err != nil { - log.Fatalf("couldn't create node service: %s", err.Error()) + log.Errorf("[METRO] Unable to initialize volume journal reconciler: %s", err.Error()) } - nodeService = ns } viper.SetConfigFile(configPath) viper.SetConfigType("yaml") viper.WatchConfig() viper.OnConfigChange(func(e fsnotify.Event) { - log.Println("Config file changed:", e.Name) + log.Infof("Config file changed: %s", e.Name) if strings.EqualFold(mode, "controller") { err := controllerService.UpdateArrays(configPath, f) @@ -180,7 +202,21 @@ func main() { InterceptorsList = append(InterceptorsList, grpc_opentracing.UnaryServerInterceptor(grpc_opentracing.WithTracer(t))) } - storageProvider := provider.New(controllerService, identityService, nodeService, InterceptorsList) + storageProvider := &gocsi.StoragePlugin{ + Controller: controllerService, + Identity: identityService, + Node: nodeService, + Interceptors: InterceptorsList, + RegisterAdditionalServers: controllerService.RegisterAdditionalServers, + + EnvVars: []string{ + // Enable request validation. + gocsi.EnvVarSpecReqValidation + "=true", + // Enable serial volume access. + gocsi.EnvVarSerialVolAccess + "=true", + }, + } + runCSIPlugin(storageProvider) } @@ -199,15 +235,13 @@ func updateDriverConfigParams(v *viper.Viper) { fmt.Printf("Read CSI_LOG_FORMAT from log configuration file, format: %s\n", logFormat) // Use JSON logger as default - if !strings.EqualFold(logFormat, "text") { - log.SetFormatter(&log.JSONFormatter{ - TimestampFormat: time.RFC3339Nano, + if strings.EqualFold(logFormat, "JSON") { + log.SetFormatter(&logrus.JSONFormatter{ + TimestampFormat: time.RFC3339, }) - } else { - log.SetFormatter(&log.TextFormatter{}) } - level := log.DebugLevel + level := csmlog.DebugLevel if v.IsSet(logLevelParam) { logLevel := v.GetString(logLevelParam) if logLevel != "" { @@ -215,15 +249,61 @@ func updateDriverConfigParams(v *viper.Viper) { fmt.Printf("Read CSI_LOG_LEVEL from log configuration file, level: %s\n", logLevel) var err error - l, err := log.ParseLevel(logLevel) + l, err := csmlog.ParseLevel(logLevel) if err != nil { - log.WithError(err).Errorf("LOG_LEVEL %s value not recognized, setting to default error: %s ", logLevel, err.Error()) + log.Errorf("LOG_LEVEL %s value not recognized, setting to default error: %s ", logLevel, err.Error()) } else { level = l } } } - log.SetLevel(level) + csmlog.SetLevel(level) +} + +func initControllerService(f fs.Interface, configPath string) (*controller.Service, error) { + cs := &controller.Service{ + Fs: f, + } + + err := cs.UpdateArrays(configPath, f) + if err != nil { + return nil, fmt.Errorf("couldn't initialize arrays in controller service: %v", err) + } + + err = cs.Init() + if err != nil { + return nil, fmt.Errorf("couldn't create controller service: %v", err) + } + + ms, err := monitor.NewMonitorService(context.Background()) + if err != nil { + return nil, fmt.Errorf("could not start monitor service: %v", err) + } + err = ms.UpdateArrays(configPath, f) + if err != nil { + return nil, fmt.Errorf("failed to initialize arrays in the monitor service: %v", err) + } + + go ms.Start(context.Background(), 1*time.Minute) + + return cs, nil +} + +func initNodeService(f fs.Interface, configPath string) (*node.Service, error) { + ns := &node.Service{ + Fs: f, + } + + err := ns.UpdateArrays(configPath, f) + if err != nil { + return nil, fmt.Errorf("couldn't initialize arrays in node service: %v", err) + } + + err = ns.Init() + if err != nil { + return nil, fmt.Errorf("couldn't create node service: %v", err) + } + return ns, nil } const usage = ` diff --git a/cmd/csi-powerstore/main_test.go b/cmd/csi-powerstore/main_test.go index e6d3625e..f158618a 100644 --- a/cmd/csi-powerstore/main_test.go +++ b/cmd/csi-powerstore/main_test.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package main import ( + "errors" "io" "os" "path/filepath" @@ -24,13 +25,22 @@ import ( "testing" "time" + "github.com/dell/csi-powerstore/v2/mocks" + "github.com/dell/csi-powerstore/v2/pkg/controller" "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/k8sutils" + "github.com/dell/csi-powerstore/v2/pkg/node" + "github.com/dell/csmlog" "github.com/dell/gocsi" "github.com/fsnotify/fsnotify" - log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" ) func TestUpdateDriverName(t *testing.T) { @@ -53,10 +63,7 @@ func TestUpdateDriverName(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - err := os.Setenv(identifiers.EnvDriverName, tc.envVar) - if err != nil { - t.Fatalf("Failed to set environment variable: %v", err) - } + t.Setenv(identifiers.EnvDriverName, tc.envVar) updateDriverName() @@ -70,25 +77,44 @@ func TestInitilizeDriverConfigParams(t *testing.T) { content := `CSI_LOG_FORMAT: "JSON"` driverConfigParams := filepath.Join(tmpDir, "driver-config-params.yaml") writeToFile(t, driverConfigParams, content) - os.Setenv(identifiers.EnvConfigParamsFilePath, driverConfigParams) + t.Setenv(identifiers.EnvConfigParamsFilePath, driverConfigParams) initilizeDriverConfigParams() - assert.Equal(t, log.DebugLevel, log.GetLevel()) + assert.Equal(t, csmlog.DebugLevel, csmlog.GetLevel()) writeToFile(t, driverConfigParams, "CSI_LOG_LEVEL: \"info\"") time.Sleep(time.Second) - assert.Equal(t, log.InfoLevel, log.GetLevel()) + assert.Equal(t, csmlog.InfoLevel, csmlog.GetLevel()) } func TestMainControllerMode(t *testing.T) { tmpDir := t.TempDir() config := copyConfigFileToTmpDir(t, "../../pkg/array/testdata/one-arr.yaml", tmpDir) + defaultK8sConfigFunc := k8sutils.InClusterConfigFunc + defaultK8sClientsetFunc := k8sutils.NewForConfigFunc + + k8sutils.InClusterConfigFunc = func() (*rest.Config, error) { + return &rest.Config{}, nil + } + k8sutils.NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return fake.NewClientset(), nil + } + + defer func() { + k8sutils.InClusterConfigFunc = defaultK8sConfigFunc + k8sutils.NewForConfigFunc = defaultK8sClientsetFunc + }() + + // Set Manifest version similar to how the image would be built. + ManifestSemver = "1.0.0" + // Set required environment variables - os.Setenv(identifiers.EnvArrayConfigFilePath, config) - os.Setenv("CSI_ENDPOINT", "mock_endpoint") - os.Setenv(identifiers.EnvDriverName, "test") - os.Setenv(identifiers.EnvDebugEnableTracing, "true") - os.Setenv("JAEGER_SERVICE_NAME", "controller-test") - os.Setenv(string(gocsi.EnvVarMode), "controller") + t.Setenv(identifiers.EnvArrayConfigFilePath, config) + t.Setenv("CSI_ENDPOINT", "mock_endpoint") + t.Setenv(identifiers.EnvDriverName, "test") + t.Setenv(identifiers.EnvDebugEnableTracing, "true") + t.Setenv("JAEGER_SERVICE_NAME", "controller-test") + t.Setenv(string(gocsi.EnvVarMode), "controller") + t.Setenv(identifiers.EnvCSMDREnabled, "true") array2 := ` - endpoint: "https://127.0.0.2/api/rest" username: "admin" @@ -102,14 +128,15 @@ func TestMainControllerMode(t *testing.T) { // Assertions require.NotNil(t, test.Controller) require.NotNil(t, test.Identity) - require.NotNil(t, test.Node) + require.Nil(t, test.Node) + require.EqualValues(t, 1, len(test.Controller.(*controller.Service).Arrays())) // Update the config file writeToFile(t, config, array2) time.Sleep(time.Second) // Assertions - require.NotNil(t, test.Controller) + require.EqualValues(t, 2, len(test.Controller.(*controller.Service).Arrays())) } defer func() { @@ -125,14 +152,28 @@ func TestMainNodeMode(t *testing.T) { tmpDir := t.TempDir() config := copyConfigFileToTmpDir(t, "../../pkg/array/testdata/one-arr.yaml", tmpDir) + defaultK8sConfigFunc := k8sutils.InClusterConfigFunc + defaultK8sClientsetFunc := k8sutils.NewForConfigFunc + + k8sutils.InClusterConfigFunc = func() (*rest.Config, error) { + return &rest.Config{}, nil + } + k8sutils.NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return fake.NewClientset(), nil + } + + defer func() { + k8sutils.InClusterConfigFunc = defaultK8sConfigFunc + k8sutils.NewForConfigFunc = defaultK8sClientsetFunc + }() + // Set required environment variables - os.Setenv(identifiers.EnvArrayConfigFilePath, config) - os.Setenv(gocsi.EnvVarMode, "node") - os.Setenv(identifiers.EnvDebugEnableTracing, "") - tempNodeIDFile, err := os.CreateTemp("", "node-id") + t.Setenv(identifiers.EnvArrayConfigFilePath, config) + t.Setenv(gocsi.EnvVarMode, "node") + t.Setenv(identifiers.EnvDebugEnableTracing, "") + tempNodeIDFile, err := os.CreateTemp(tmpDir, "node-id") require.NoError(t, err) - defer os.Remove(tempNodeIDFile.Name()) - os.Setenv("X_CSI_POWERSTORE_NODE_ID_PATH", tempNodeIDFile.Name()) + t.Setenv("X_CSI_POWERSTORE_NODE_ID_PATH", tempNodeIDFile.Name()) array2 := ` - endpoint: "https://127.0.0.2/api/rest" username: "admin" @@ -144,16 +185,17 @@ func TestMainNodeMode(t *testing.T) { runCSIPlugin = func(test *gocsi.StoragePlugin) { // Assertions - require.NotNil(t, test.Controller) + require.Nil(t, test.Controller) require.NotNil(t, test.Identity) require.NotNil(t, test.Node) + require.EqualValues(t, 1, len(test.Node.(*node.Service).Arrays())) // Update the config file writeToFile(t, config, array2) time.Sleep(time.Second) // Assertions - require.NotNil(t, test.Node) + require.EqualValues(t, 2, len(test.Node.(*node.Service).Arrays())) } defer func() { @@ -212,22 +254,142 @@ func TestUpdateDriverConfigParams(t *testing.T) { assert.Equal(t, "text", logFormat) updateDriverConfigParams(v) - level := log.GetLevel() + level := csmlog.GetLevel() - assert.Equal(t, log.DebugLevel, level) + assert.Equal(t, csmlog.DebugLevel, level) v.Set("CSI_LOG_FORMAT", "json") v.Set("CSI_LOG_LEVEL", "info") updateDriverConfigParams(v) - level = log.GetLevel() + level = csmlog.GetLevel() - assert.Equal(t, log.InfoLevel, level) - logFormatter, ok := log.StandardLogger().Formatter.(*log.JSONFormatter) - assert.True(t, ok) - assert.Equal(t, time.RFC3339Nano, logFormatter.TimestampFormat) + assert.Equal(t, csmlog.InfoLevel, level) + logFormatter := &csmlog.MyTextFormatter{ + Base: &logrus.TextFormatter{ + TimestampFormat: time.RFC3339, + }, + } + assert.Equal(t, time.RFC3339, logFormatter.Base.TimestampFormat) v.Set("CSI_LOG_LEVEL", "notalevel") updateDriverConfigParams(v) - level = log.GetLevel() - assert.Equal(t, log.DebugLevel, level) + level = csmlog.GetLevel() + assert.Equal(t, csmlog.DebugLevel, level) +} + +func Test_initControllerService(t *testing.T) { + tests := []struct { + name string // description of this test case + // Named input parameters for target function. + init func() + f func() fs.Interface + configPath string + want *controller.Service + wantErr bool + }{ + { + name: "fail to update arrays", + init: func() {}, + f: func() fs.Interface { + fs := &mocks.FsInterface{} + fs.On("ReadFile", ".").Return([]byte{}, errors.New("read error")) + return fs + }, + configPath: "", + want: nil, + wantErr: true, + }, + { + name: "fail to initialize the controller service", + init: func() { + tempNewForConfigFunc := k8sutils.NewForConfigFunc + k8sutils.NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return nil, errors.New("new for config error") + } + t.Cleanup(func() { + k8sutils.NewForConfigFunc = tempNewForConfigFunc + }) + }, + f: func() fs.Interface { + fs := &mocks.FsInterface{} + fs.On("ReadFile", "/some/config.yaml").Return([]byte{}, nil) + return fs + }, + configPath: "/some/config.yaml", + want: nil, + wantErr: true, + }, + { + name: "fail to initialize the monitor service arrays", + init: func() { + tempNewForConfigFunc := k8sutils.NewForConfigFunc + k8sutils.NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return fake.NewClientset(), nil + } + tempInClusterConfigFunc := k8sutils.InClusterConfigFunc + k8sutils.InClusterConfigFunc = func() (*rest.Config, error) { + return nil, nil + } + t.Cleanup(func() { + k8sutils.NewForConfigFunc = tempNewForConfigFunc + k8sutils.InClusterConfigFunc = tempInClusterConfigFunc + }) + }, + f: func() fs.Interface { + fs := &mocks.FsInterface{} + fs.On("ReadFile", ".").Once().Return([]byte{}, nil) + fs.On("ReadFile", ".").Once().Return([]byte{}, errors.New("monitor read error")) + return fs + }, + configPath: "", + want: nil, + wantErr: true, + }, + { + name: "success", + init: func() { + tempNewForConfigFunc := k8sutils.NewForConfigFunc + k8sutils.NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return fake.NewClientset(), nil + } + tempInClusterConfigFunc := k8sutils.InClusterConfigFunc + k8sutils.InClusterConfigFunc = func() (*rest.Config, error) { + return nil, nil + } + t.Cleanup(func() { + k8sutils.NewForConfigFunc = tempNewForConfigFunc + k8sutils.InClusterConfigFunc = tempInClusterConfigFunc + }) + }, + f: func() fs.Interface { + fs := &mocks.FsInterface{} + fs.On("ReadFile", ".").Return([]byte{}, nil) + return fs + }, + configPath: "", + want: &controller.Service{ + Fs: &mocks.FsInterface{}, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.init() + got, gotErr := initControllerService(tt.f(), tt.configPath) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("initControllerService() failed: %v", gotErr) + } + return + } + if tt.wantErr { + t.Fatal("initControllerService() succeeded unexpectedly") + } + + if got == nil { + t.Error("initControllerService() expected a service struct but got nil") + } + }) + } } diff --git a/core/semver/semver.go b/core/semver/semver.go index e0fe3fa6..af196c09 100644 --- a/core/semver/semver.go +++ b/core/semver/semver.go @@ -357,7 +357,7 @@ var OSExit = func(code int) { // GetExitError is a wrapper around exec.ExitError var GetExitError = func(err error) (e *exec.ExitError, ok bool) { e, ok = err.(*exec.ExitError) - return + return e, ok } // GetStatusError is a wrapper around syscall.WaitStatus diff --git a/dell-csi-helm-installer/csi-install.sh b/dell-csi-helm-installer/csi-install.sh index af488242..14c8756a 100755 --- a/dell-csi-helm-installer/csi-install.sh +++ b/dell-csi-helm-installer/csi-install.sh @@ -141,11 +141,21 @@ function install_driver() { fi HELMOUTPUT="/tmp/csi-install.$$.out" + + # install all of the helm dependencies + log step "Installing helm dependencies" + run_command helm dependency build "${DRIVERDIR}/${DRIVER}" >>"${HELMOUTPUT}" 2>&1 + if [ $? -ne 0 ]; then + cat "${HELMOUTPUT}" + log error "Failed to install ${DRIVER} helm dependencies" + fi + + # install the driver run_command helm ${1} \ --set openshift=${OPENSHIFT} \ --values "${VALUES}" \ --namespace ${NS} "${RELEASE}" \ - "${DRIVERDIR}/${DRIVER}" >"${HELMOUTPUT}" 2>&1 + "${DRIVERDIR}/${DRIVER}" >>"${HELMOUTPUT}" 2>&1 if [ $? -ne 0 ]; then cat "${HELMOUTPUT}" diff --git a/dell-csi-helm-installer/csi-uninstall.sh b/dell-csi-helm-installer/csi-uninstall.sh index 08dd5857..b696c694 100755 --- a/dell-csi-helm-installer/csi-uninstall.sh +++ b/dell-csi-helm-installer/csi-uninstall.sh @@ -70,6 +70,18 @@ function check_for_driver() { fi } +# Since helm does not automatically clean up crds on an uninstall. +# This step must be done manually. +# check here for more information https://helm.sh/docs/chart_best_practices/custom_resource_definitions +function remove_crd_after_uninstall() { + # Remove the volume journals crd + decho "Removal of the CSI PowerStore Custom Resource Definitions..." + run_command kubectl delete crd volumejournals.dr.storage.dell.com + if [ $? -ne 0 ]; then + decho "Removal of the CSI PowerStore Custom Resource Definition (volumejournals.dr.storage.dell.com) failed" + fi +} + # get the list of valid CSI Drivers, this will be the list of directories in drivers/ that contain helm charts DRIVERDIR="${SCRIPTDIR}/../helm-charts/charts" @@ -125,6 +137,8 @@ if [ $? -ne 0 ]; then exit 1 fi +remove_crd_after_uninstall + decho "Removal of the CSI Driver is in progress." decho "It may take a few minutes for all pods to terminate." diff --git a/dell-csi-helm-installer/verify.sh b/dell-csi-helm-installer/verify.sh index 6843ea79..b18a8824 100755 --- a/dell-csi-helm-installer/verify.sh +++ b/dell-csi-helm-installer/verify.sh @@ -355,6 +355,27 @@ function verify_alpha_snap_resources() { check_error error } +# verify that the requirements for disaster recover support exist +function verify_dr_requirements() { + log step "Verifying disaster recover support" + decho + log arrow + log smart_step "Verifying that disaster recover CRDs are available" "small" + + error=0 + # check for the CRDs. These are required for installation + CRDS=("VolumeJournal") + for C in "${CRDS[@]}"; do + # Verify if DR related CRDs are there on the system. + run_command kubectl explain ${C} 2>&1 >/dev/null + if [ $? -ne 0 ]; then + error=1 + found_error "The CRD for ${C} is not installed. These need to be installed by the Kubernetes administrator" + fi + done + check_error error +} + # verify that the requirements for snapshot support exist function verify_snap_requirements() { log step "Verifying snapshot support" diff --git a/docker.mk b/docker.mk deleted file mode 100644 index 9e8650b1..00000000 --- a/docker.mk +++ /dev/null @@ -1,66 +0,0 @@ -# -# -# Copyright © 2020-2023 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -# for variables override --include vars.mk - -# Includes the following generated file to get semantic version information -include semver.mk -ifdef NOTES - RELNOTE="$(NOTES)" -else - RELNOTE= -endif - -ifeq ($(IMAGETAG),) -IMAGETAG=v$(MAJOR).$(MINOR).$(PATCH)$(RELNOTE) -endif - -ifndef DOCKER_REGISTRY - DOCKER_REGISTRY=dellemc -endif - -ifndef DOCKER_IMAGE_NAME - DOCKER_IMAGE_NAME=csi-powerstore -endif - -# set the GOVERSION -export GOVERSION="1.21" - -# figure out if podman or docker should be used (use podman if found) -ifneq (, $(shell which podman 2>/dev/null)) - BUILDER=podman -else - BUILDER=docker -endif - -docker: download-csm-common - $(eval include csm-common.mk) - $(BUILDER) build --pull -t "$(DOCKER_REGISTRY)/$(DOCKER_IMAGE_NAME):$(IMAGETAG)" --build-arg GOIMAGE=$(DEFAULT_GOIMAGE) --build-arg BASEIMAGE=$(CSM_BASEIMAGE) . - -docker-no-cache: download-csm-common - $(eval include csm-common.mk) - $(BUILDER) build --pull --no-cache -t "$(DOCKER_REGISTRY)/$(DOCKER_IMAGE_NAME):$(IMAGETAG)" --build-arg GOIMAGE=$(DEFAULT_GOIMAGE) --build-arg BASEIMAGE=$(CSM_BASEIMAGE) . - -push: - $(BUILDER) push "$(DOCKER_REGISTRY)/$(DOCKER_IMAGE_NAME):$(IMAGETAG)" - -download-csm-common: - curl -O -L https://raw.githubusercontent.com/dell/csm/main/config/csm-common.mk - -tag: - -git tag -d $(IMAGETAG) - git tag -a -m $(TAGMSG) $(IMAGETAG) diff --git a/go.mod b/go.mod index 60729be2..3945eaf3 100644 --- a/go.mod +++ b/go.mod @@ -1,123 +1,137 @@ module github.com/dell/csi-powerstore/v2 -go 1.25 +go 1.25.0 require ( + github.com/dell/csi-metadata-retriever v1.13.0 + github.com/dell/csm-dr v1.0.0 + github.com/dell/csmlog v1.0.0 + github.com/dell/dell-csi-extensions/common v1.10.0 + github.com/dell/dell-csi-extensions/podmon v1.10.0 + github.com/dell/dell-csi-extensions/replication v1.13.0 + github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.8.1 + github.com/dell/gobrick v1.16.0 + github.com/dell/gocsi v1.16.0 + github.com/dell/gofsutil v1.21.0 + github.com/dell/goiscsi v1.14.0 + github.com/dell/gonvme v1.13.0 + github.com/dell/gopowerstore v1.21.0 github.com/akutz/gosync v0.1.0 github.com/apparentlymart/go-cidr v1.1.0 - github.com/container-storage-interface/spec v1.6.0 - github.com/dell/csi-metadata-retriever v1.12.0 - github.com/dell/csm-sharednfs v1.0.1-0.20250917153118-9094f4bc87d9 - github.com/dell/dell-csi-extensions/common v1.9.0 - github.com/dell/dell-csi-extensions/podmon v1.9.0 - github.com/dell/dell-csi-extensions/replication v1.12.0 - github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.8.1 - github.com/dell/gobrick v1.15.0 - github.com/dell/gocsi v1.15.0 - github.com/dell/gofsutil v1.20.0 - github.com/dell/goiscsi v1.13.0 - github.com/dell/gonvme v1.12.0 - github.com/dell/gopowerstore v1.20.1 + github.com/container-storage-interface/spec v1.7.0 github.com/fsnotify/fsnotify v1.9.0 - github.com/go-openapi/strfmt v0.23.0 + github.com/go-openapi/strfmt v0.25.0 + github.com/gogo/protobuf v1.3.2 github.com/golang/mock v1.6.0 + github.com/golang/protobuf v1.5.4 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/kubernetes-csi/csi-lib-utils v0.11.0 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.38.0 + github.com/onsi/gomega v1.39.0 github.com/opentracing/opentracing-go v1.2.0 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/viper v1.20.0 + github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible go.uber.org/mock v0.6.0 - golang.org/x/net v0.43.0 - google.golang.org/grpc v1.75.0 - google.golang.org/protobuf v1.36.6 - gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.34.0 - k8s.io/apimachinery v0.34.0 - k8s.io/client-go v0.34.0 + golang.org/x/net v0.49.0 + google.golang.org/grpc v1.78.0 + google.golang.org/protobuf v1.36.11 + k8s.io/api v0.35.0 + k8s.io/apimachinery v0.35.0 + k8s.io/client-go v0.35.0 + k8s.io/component-helpers v0.35.0 + sigs.k8s.io/controller-runtime v0.22.4 + sigs.k8s.io/yaml v1.6.0 ) require ( github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect - github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/bombsimon/logrusr/v4 v4.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/coreos/go-systemd/v22 v22.6.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-logr/logr v1.4.3 // indirect - github.com/go-openapi/errors v0.22.0 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.21.0 // indirect - github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-openapi/errors v0.22.4 // indirect + github.com/go-openapi/jsonpointer v0.22.3 // indirect + github.com/go-openapi/jsonreference v0.21.3 // indirect + github.com/go-openapi/swag v0.25.4 // indirect + github.com/go-openapi/swag/cmdutils v0.25.4 // indirect + github.com/go-openapi/swag/conv v0.25.4 // indirect + github.com/go-openapi/swag/fileutils v0.25.4 // indirect + github.com/go-openapi/swag/jsonname v0.25.4 // indirect + github.com/go-openapi/swag/jsonutils v0.25.4 // indirect + github.com/go-openapi/swag/loading v0.25.4 // indirect + github.com/go-openapi/swag/mangling v0.25.4 // indirect + github.com/go-openapi/swag/netutils v0.25.4 // indirect + github.com/go-openapi/swag/stringutils v0.25.4 // indirect + github.com/go-openapi/swag/typeutils v0.25.4 // indirect + github.com/go-openapi/swag/yamlutils v0.25.4 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect - github.com/josharian/intern v1.0.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.11 // indirect - github.com/mailru/easyjson v0.9.0 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nxadm/tail v1.4.11 // indirect github.com/oklog/ulid v1.3.1 // indirect - github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.20.5 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.62.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - github.com/sagikazarmark/locafero v0.7.0 // indirect - github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.12.0 // indirect - github.com/spf13/cast v1.7.1 // indirect - github.com/spf13/pflag v1.0.9 // indirect - github.com/stretchr/objx v0.5.2 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.67.4 // indirect + github.com/prometheus/procfs v0.19.2 // indirect + github.com/sagikazarmark/locafero v0.12.0 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/stretchr/objx v0.5.3 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/x448/float16 v0.8.4 // indirect - go.etcd.io/etcd/api/v3 v3.6.1 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.6.1 // indirect - go.etcd.io/etcd/client/v3 v3.6.1 // indirect - go.mongodb.org/mongo-driver v1.17.2 // indirect - go.opentelemetry.io/otel v1.37.0 // indirect - go.opentelemetry.io/otel/trace v1.37.0 // indirect + go.etcd.io/etcd/api/v3 v3.6.6 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.6.6 // indirect + go.etcd.io/etcd/client/v3 v3.6.6 // indirect + go.mongodb.org/mongo-driver v1.17.6 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect - go.yaml.in/yaml/v2 v2.4.2 // indirect + go.uber.org/zap v1.27.1 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.17.0 // indirect - golang.org/x/sys v0.36.0 // indirect - golang.org/x/term v0.34.0 // indirect - golang.org/x/text v0.28.0 // indirect - golang.org/x/time v0.9.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + golang.org/x/oauth2 v0.33.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/term v0.39.0 // indirect + golang.org/x/text v0.33.0 // indirect + golang.org/x/time v0.14.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - k8s.io/component-base v0.32.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.34.2 // indirect + k8s.io/component-base v0.34.2 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect - k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect - sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e // indirect + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.1 // indirect ) diff --git a/go.sum b/go.sum index 8c593b60..774e9111 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,32 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/dell/csi-metadata-retriever v1.13.0 h1:6eb/VaVoY3NVGXUppMJEDZirsH1y8Ki/SHkJCkmI+L8= +github.com/dell/csi-metadata-retriever v1.13.0/go.mod h1:ZiOG9J1MjTUslncrbfgZ2v8sxdOB6ZdRtVjRCAQ0tmc= +github.com/dell/csm-dr v1.0.0 h1:LW6MHU/YYiaxI3yUV+riEIWGbPV7UW0OgD4AWCct/HM= +github.com/dell/csm-dr v1.0.0/go.mod h1:/yKWAo804JSVNzUYLPc43v7dE0E0eH53P02Kil7HMRw= +github.com/dell/csmlog v1.0.0 h1:EzW+nMJBD0QTNP88OoaAJUOMVilS9cWkO248BTcJt/4= +github.com/dell/csmlog v1.0.0/go.mod h1:7rBzSv9xF5t233+J+9vkStjFsmyYO3L/B9tDTy3+9ZU= +github.com/dell/dell-csi-extensions/common v1.10.0 h1:WIFPWVEBUyzOTCOPAlcgQsiRRGAufyKJYbIATbNZXIY= +github.com/dell/dell-csi-extensions/common v1.10.0/go.mod h1:zRHzmPX5SQQnqQ1LEIxG4hYqLBeQOSiD8TkEhU0eWTY= +github.com/dell/dell-csi-extensions/podmon v1.10.0 h1:YeM9OmgJHE+n6aNaeEC96EuVev5x3pddggcM7Ws7pkk= +github.com/dell/dell-csi-extensions/podmon v1.10.0/go.mod h1:+g7fdyw1Zx74NBJQgi1BCtsywqk37MJd9JN86IPJJu0= +github.com/dell/dell-csi-extensions/replication v1.13.0 h1:DSpoZ3vX65a3KDxUv0OinLkY2qUAQtRX3E1c1e3fnvA= +github.com/dell/dell-csi-extensions/replication v1.13.0/go.mod h1:aJBwd55amqbY3kk8SG7NjwH7nxBscceDwc1rKesUG1g= +github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.8.1 h1:W0UcLCZ8qyJ+NBRDfrZefN+fMs2i73ydkIsq6RjP7bM= +github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.8.1/go.mod h1:C4Ji1GCEayZ733hcHkvM6JDcboWJjk43H7xp30a/trc= +github.com/dell/gobrick v1.16.0 h1:z/a9qXnT3hx3D4I+SJUMnIgJtcCx0j3gzmPPDUWtoYs= +github.com/dell/gobrick v1.16.0/go.mod h1:9uoH8EsNi9yAsUZj2gZFgB5kqdlyvArqx0tYC7Qg9IM= +github.com/dell/gocsi v1.16.0 h1:avhQPD11rYzT6/dPxpZfFsJV+T/T0x1GJqqbco45W8c= +github.com/dell/gocsi v1.16.0/go.mod h1:Fz5dQv/kWf5Y1EXZEzxLBQSsnW2HE/WY95R0WCDQLO4= +github.com/dell/gofsutil v1.21.0 h1:SeusAYjiO/1ogvg/TapvCyHcrM9z+OvdaMU5i9Ijn3M= +github.com/dell/gofsutil v1.21.0/go.mod h1:qBGEz1wMOtqTODuJfiBZhUHT0JjexBblu2oa+sEclNs= +github.com/dell/goiscsi v1.14.0 h1:kNDqOlpJ3cLSJh7Hfyn/Kz/FMCKHzV0s/xx4EqnelFw= +github.com/dell/goiscsi v1.14.0/go.mod h1:SCSC8dJCqTosU7SspaoLv6ICTKNEz08rt/I8nZ3+ptc= +github.com/dell/gonvme v1.13.0 h1:j8A1BzYA48gelih3xWd/J6LQ71CbC8Lbdyv0jG8uUNU= +github.com/dell/gonvme v1.13.0/go.mod h1:L5K7V4JZTf12m3k2wdwKwP+/eA6pr8DvlCsJU1QTGOQ= +github.com/dell/gopowerstore v1.21.0 h1:CjuBg6OLHzNh0lsWIh7mVDKqFvZIf65c5AT+BYP9L3s= +github.com/dell/gopowerstore v1.21.0/go.mod h1:MxuIIOHhcaIgCjzXR4DT5NSRe+XIr/5xWM3dH47fmUk= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -35,6 +61,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -54,8 +82,6 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= -github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -67,6 +93,8 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bombsimon/logrusr/v4 v4.1.0 h1:uZNPbwusB0eUXlO8hIUwStE6Lr5bLN6IgYgG+75kuh4= +github.com/bombsimon/logrusr/v4 v4.1.0/go.mod h1:pjfHC5e59CvjTBIU3V3sGhFWFAnsnhOR03TRc6im0l8= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -81,16 +109,16 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/container-storage-interface/spec v1.5.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= -github.com/container-storage-interface/spec v1.6.0 h1:vwN9uCciKygX/a0toYryoYD5+qI9ZFeAMuhEEKO+JBA= -github.com/container-storage-interface/spec v1.6.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= +github.com/container-storage-interface/spec v1.7.0 h1:gW8eyFQUZWWrMWa8p1seJ28gwDoN5CVJ4uAbQ+Hdycw= +github.com/container-storage-interface/spec v1.7.0/go.mod h1:JYuzLqr9VVNoDJl44xp/8fmCOvWPDKzuGTwCoklhuqk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo= +github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -99,30 +127,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dell/csi-metadata-retriever v1.12.0 h1:54z7Sm2XCgTfiEI8lcGwg5P24brEnfylqGvMzQiWKWo= -github.com/dell/csi-metadata-retriever v1.12.0/go.mod h1:TsLkzdvdsp/EBT2ffeTw3Sq/v2bSWzIIvfUfhhYQBZg= -github.com/dell/csm-sharednfs v1.0.1-0.20250917153118-9094f4bc87d9 h1:anWPGeqwPewoxYYqkJO1+qWBgSjwHGAm6uy66lMMtyQ= -github.com/dell/csm-sharednfs v1.0.1-0.20250917153118-9094f4bc87d9/go.mod h1:T8SgcJiGYQcmza8k/RKIimd908NSPDojrzLPwf2EpLs= -github.com/dell/dell-csi-extensions/common v1.9.0 h1:H1NXBYlJZ+XTCe4tSXo94Lvg8HD2wgt6ywql3kVrG34= -github.com/dell/dell-csi-extensions/common v1.9.0/go.mod h1:DA9lX2BX3fdshR40IaXfokDrIKo9a32QShcTlAqhf+c= -github.com/dell/dell-csi-extensions/podmon v1.9.0 h1:AYE3n6o6jB3Sh0uce65JPmir3FPxvqSW/21/bGqRhvY= -github.com/dell/dell-csi-extensions/podmon v1.9.0/go.mod h1:jz846RAruY/m25uBbZVYcr8vp7wmKakbjOuUBwpY0Ls= -github.com/dell/dell-csi-extensions/replication v1.12.0 h1:jOdaZsoGHWX9SsqgH+2v9cIJSAVLF1SnKKHtdiF5Ywc= -github.com/dell/dell-csi-extensions/replication v1.12.0/go.mod h1:nyPBfbMOpboVI/cYLOFJhv0LADGSvHwDcF4AxZau/go= -github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.8.1 h1:NnS/P2OpwMlQ70fwls/KVVfe8z8op4b7nXArv8CDsqk= -github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.8.1/go.mod h1:rlJGlmp1NI8gU52HYKY2cy13TbSWvt9p5VAG2RLbkQs= -github.com/dell/gobrick v1.15.0 h1:0BAjAjDxnpX2b3aIuemhhWsnaOS09P1JeFA8vibDlmI= -github.com/dell/gobrick v1.15.0/go.mod h1:5GhPEB6AE5dWmKZLSMl5QT8BZBSkcolI7GBej1wo8XY= -github.com/dell/gocsi v1.15.0 h1:SXBtiNTb3iTHms4WRoewwdJaItOY8XaaxBjkTYy8o5Q= -github.com/dell/gocsi v1.15.0/go.mod h1:u8+NcCB2rWr79Dx63GWUo3TsAJj/RSlRoimTrp6BZiM= -github.com/dell/gofsutil v1.20.0 h1:jkQrOb4sSxEUcPTAbyLBABMBf+7vBC6g+yzxTGb0Ozw= -github.com/dell/gofsutil v1.20.0/go.mod h1:kKFZSYY0tF5lx/U6UhSAqLxKnNESd0hT4gJ4PlYXSB8= -github.com/dell/goiscsi v1.13.0 h1:4+uB+uJQmJ91yN7wy38sLsr5S/lqL3/tVboLOh0sg38= -github.com/dell/goiscsi v1.13.0/go.mod h1:1IPCAavfm6T9BzKS0QYfBlJz7X+AfYPYjH4G84TvJP4= -github.com/dell/gonvme v1.12.0 h1:KLOr+v+1kn/sz26CFTAkFrR1Ti4aZ37i1Mlxp1hBXYs= -github.com/dell/gonvme v1.12.0/go.mod h1:ETLwyr+OG3DYfzdlMKCv5PjeDfj+JIxV2xrbHBTg2lk= -github.com/dell/gopowerstore v1.20.1 h1:Z2N5eWVBG+SrwtPMEXVeGOJ3jquhkrZ6PyCFpwfkDSg= -github.com/dell/gopowerstore v1.20.1/go.mod h1:PNkGw7gZUyyjdJdZdKKvGve30DMse/SzOQqUkVN8HCM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= @@ -130,15 +134,18 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= -github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= @@ -169,19 +176,49 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= -github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/errors v0.22.4 h1:oi2K9mHTOb5DPW2Zjdzs/NIvwi2N3fARKaTJLdNabaM= +github.com/go-openapi/errors v0.22.4/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonpointer v0.22.3 h1:dKMwfV4fmt6Ah90zloTbUKWMD+0he+12XYAsPotrkn8= +github.com/go-openapi/jsonpointer v0.22.3/go.mod h1:0lBbqeRsQ5lIanv3LHZBrmRGHLHcQoOXQnf88fHlGWo= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= -github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= -github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c= -github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4= +github.com/go-openapi/jsonreference v0.21.3 h1:96Dn+MRPa0nYAR8DR1E03SblB5FJvh7W6krPI0Z7qMc= +github.com/go-openapi/jsonreference v0.21.3/go.mod h1:RqkUP0MrLf37HqxZxrIAtTWW4ZJIK1VzduhXYBEeGc4= +github.com/go-openapi/strfmt v0.25.0 h1:7R0RX7mbKLa9EYCTHRcCuIPcaqlyQiWNPTXwClK0saQ= +github.com/go-openapi/strfmt v0.25.0/go.mod h1:nNXct7OzbwrMY9+5tLX4I21pzcmE6ccMGXl3jFdPfn8= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-openapi/swag v0.25.4 h1:OyUPUFYDPDBMkqyxOTkqDYFnrhuhi9NR6QVUvIochMU= +github.com/go-openapi/swag v0.25.4/go.mod h1:zNfJ9WZABGHCFg2RnY0S4IOkAcVTzJ6z2Bi+Q4i6qFQ= +github.com/go-openapi/swag/cmdutils v0.25.4 h1:8rYhB5n6WawR192/BfUu2iVlxqVR9aRgGJP6WaBoW+4= +github.com/go-openapi/swag/cmdutils v0.25.4/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= +github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4= +github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU= +github.com/go-openapi/swag/fileutils v0.25.4 h1:2oI0XNW5y6UWZTC7vAxC8hmsK/tOkWXHJQH4lKjqw+Y= +github.com/go-openapi/swag/fileutils v0.25.4/go.mod h1:cdOT/PKbwcysVQ9Tpr0q20lQKH7MGhOEb6EwmHOirUk= +github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI= +github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= +github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA= +github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4 h1:IACsSvBhiNJwlDix7wq39SS2Fh7lUOCJRmx/4SN4sVo= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4/go.mod h1:Mt0Ost9l3cUzVv4OEZG+WSeoHwjWLnarzMePNDAOBiM= +github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s= +github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE= +github.com/go-openapi/swag/mangling v0.25.4 h1:2b9kBJk9JvPgxr36V23FxJLdwBrpijI26Bx5JH4Hp48= +github.com/go-openapi/swag/mangling v0.25.4/go.mod h1:6dxwu6QyORHpIIApsdZgb6wBk/DPU15MdyYj/ikn0Hg= +github.com/go-openapi/swag/netutils v0.25.4 h1:Gqe6K71bGRb3ZQLusdI8p/y1KLgV4M/k+/HzVSqT8H0= +github.com/go-openapi/swag/netutils v0.25.4/go.mod h1:m2W8dtdaoX7oj9rEttLyTeEFFEBvnAx9qHd5nJEBzYg= +github.com/go-openapi/swag/stringutils v0.25.4 h1:O6dU1Rd8bej4HPA3/CLPciNBBDwZj9HiEpdVsb8B5A8= +github.com/go-openapi/swag/stringutils v0.25.4/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0= +github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv42wAJSN91wRw= +github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE= +github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw= +github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc= +github.com/go-openapi/testify/enable/yaml/v2 v2.0.2 h1:0+Y41Pz1NkbTHz8NngxTuAXxEodtNSI1WG1c/m5Akw4= +github.com/go-openapi/testify/enable/yaml/v2 v2.0.2/go.mod h1:kme83333GCtJQHXQ8UKX3IBZu6z8T5Dvy5+CW3NLUUg= +github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= +github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -189,13 +226,12 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= -github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -232,8 +268,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= -github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= +github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -245,6 +281,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -274,13 +312,13 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDa github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= -github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= -github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0 h1:FbSCl+KggFl+Ocym490i/EyXF4lPgLoUtcSWquBM0Rs= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0/go.mod h1:qOchhhIlmRcqk/O9uCo/puJlyo07YINaIqdZfZG3Jkc= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -310,8 +348,6 @@ github.com/jarcoal/httpmock v1.4.0/go.mod h1:ftW1xULwo+j0R0JJkJIIi7UKigZUXCLLany github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -327,8 +363,8 @@ github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -347,8 +383,6 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+ github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= -github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -362,8 +396,6 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -393,20 +425,20 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= -github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.38.0 h1:c/WX+w8SLAinvuKKQFh77WEucCnPk4j2OTUr7lt7BeY= -github.com/onsi/gomega v1.38.0/go.mod h1:OcXcwId0b9QsE7Y49u+BTrL4IdKOBOKnD6VQNTJEB6o= +github.com/onsi/gomega v1.39.0 h1:y2ROC3hKFmQZJNFeGAMeHZKkjBL65mIZcvrLQBF9k6Q= +github.com/onsi/gomega v1.39.0/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -421,38 +453,38 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= -github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc= +github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= +github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= -github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= +github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4= +github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -465,31 +497,29 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= -github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= -github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= -github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= -github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.20.0 h1:zrxIyR3RQIOsarIrgL8+sAvALXul9jeEPa06Y0Ph6vY= -github.com/spf13/viper v1.20.0/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -503,50 +533,50 @@ github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSW github.com/thecodeteam/gosync v0.1.0 h1:RcD9owCaiK0Jg1rIDPgirdcLCL1jCD6XlDVSg0MfHmE= github.com/thecodeteam/gosync v0.1.0/go.mod h1:43QHsngcnWc8GE1aCmi7PEypslflHjCzXFleuWKEb00= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= +github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= -go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.etcd.io/etcd/api/v3 v3.6.1 h1:yJ9WlDih9HT457QPuHt/TH/XtsdN2tubyxyQHSHPsEo= -go.etcd.io/etcd/api/v3 v3.6.1/go.mod h1:lnfuqoGsXMlZdTJlact3IB56o3bWp1DIlXPIGKRArto= -go.etcd.io/etcd/client/pkg/v3 v3.6.1 h1:CxDVv8ggphmamrXM4Of8aCC8QHzDM4tGcVr9p2BSoGk= -go.etcd.io/etcd/client/pkg/v3 v3.6.1/go.mod h1:aTkCp+6ixcVTZmrJGa7/Mc5nMNs59PEgBbq+HCmWyMc= -go.etcd.io/etcd/client/v3 v3.6.1 h1:KelkcizJGsskUXlsxjVrSmINvMMga0VWwFF0tSPGEP0= -go.etcd.io/etcd/client/v3 v3.6.1/go.mod h1:fCbPUdjWNLfx1A6ATo9syUmFVxqHH9bCnPLBZmnLmMY= -go.etcd.io/etcd/pkg/v3 v3.6.1 h1:Qpshk3/SLra217k7FxcFGaH2niFAxFf1Dug57f0IUiw= -go.etcd.io/etcd/pkg/v3 v3.6.1/go.mod h1:nS0ahQoZZ9qXjQAtYGDt80IEHKl9YOF7mv6J0lQmBoQ= -go.etcd.io/etcd/server/v3 v3.6.1 h1:Y/mh94EeImzXyTBIMVgR0v5H+ANtRFDY4g1s5sxOZGE= -go.etcd.io/etcd/server/v3 v3.6.1/go.mod h1:nCqJGTP9c2WlZluJB59j3bqxZEI/GYBfQxno0MguVjE= +go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= +go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= +go.etcd.io/etcd/api/v3 v3.6.6 h1:mcaMp3+7JawWv69p6QShYWS8cIWUOl32bFLb6qf8pOQ= +go.etcd.io/etcd/api/v3 v3.6.6/go.mod h1:f/om26iXl2wSkcTA1zGQv8reJRSLVdoEBsi4JdfMrx4= +go.etcd.io/etcd/client/pkg/v3 v3.6.6 h1:uoqgzSOv2H9KlIF5O1Lsd8sW+eMLuV6wzE3q5GJGQNs= +go.etcd.io/etcd/client/pkg/v3 v3.6.6/go.mod h1:YngfUVmvsvOJ2rRgStIyHsKtOt9SZI2aBJrZiWJhCbI= +go.etcd.io/etcd/client/v3 v3.6.6 h1:G5z1wMf5B9SNexoxOHUGBaULurOZPIgGPsW6CN492ec= +go.etcd.io/etcd/client/v3 v3.6.6/go.mod h1:36Qv6baQ07znPR3+n7t+Rk5VHEzVYPvFfGmfF4wBHV8= +go.etcd.io/etcd/pkg/v3 v3.6.6 h1:wylOivS/UxXTZ0Le5fOdxCjatW5ql9dcWEggQQHSorw= +go.etcd.io/etcd/pkg/v3 v3.6.6/go.mod h1:9TKZL7WUEVHXYM3srP3ESZfIms34s1G72eNtWA9YKg4= +go.etcd.io/etcd/server/v3 v3.6.6 h1:YSRWGJPzU+lIREwUQI4MfyLZrkUyzjJOVpMxJvZePaY= +go.etcd.io/etcd/server/v3 v3.6.6/go.mod h1:A1OQ1x3PaiENDLywMjCiMwV1pwJSpb0h9Z5ORP2dv6I= go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ= go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo= -go.mongodb.org/mongo-driver v1.17.2 h1:gvZyk8352qSfzyZ2UMWcpDpMSGEr1eqE4T793SqyhzM= -go.mongodb.org/mongo-driver v1.17.2/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss= +go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 h1:rgMkmiGfix9vFJDcDi1PK8WEQP4FLQwLDfhp5ZLpFeE= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0/go.mod h1:ijPqXp5P6IRRByFVVg9DY8P5HkxkHE5ARIa+86aXPf4= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= @@ -554,19 +584,19 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqx go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= -go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= -go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= -go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= @@ -574,8 +604,6 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= -go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= @@ -588,10 +616,10 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= -go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -603,8 +631,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= -golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= +golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -641,6 +669,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= +golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -672,15 +702,15 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo= +golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -690,8 +720,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -739,13 +769,13 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= -golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= +golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -753,14 +783,14 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= -golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -803,12 +833,14 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= -golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= +golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= +gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= @@ -853,10 +885,10 @@ google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 h1:FiusG7LWj+4byqhbvmB+Q93B/mOxJLN2DTozDuZm4EU= -google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:kXqgZtrWaf6qS3jZOCnCH7WYfrvFjkC51bM8fz3RsCA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= +google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -870,8 +902,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= -google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -884,8 +916,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -894,8 +926,8 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= @@ -928,40 +960,46 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.22.0/go.mod h1:0AoXXqst47OI/L0oGKq9DG61dvGRPXs7X4/B7KyjBCU= -k8s.io/api v0.34.0 h1:L+JtP2wDbEYPUeNGbeSa/5GwFtIA662EmT2YSLOkAVE= -k8s.io/api v0.34.0/go.mod h1:YzgkIzOOlhl9uwWCZNqpw6RJy9L2FK4dlJeayUoydug= +k8s.io/api v0.35.0 h1:iBAU5LTyBI9vw3L5glmat1njFK34srdLmktWwLTprlY= +k8s.io/api v0.35.0/go.mod h1:AQ0SNTzm4ZAczM03QH42c7l3bih1TbAXYo0DkF8ktnA= +k8s.io/apiextensions-apiserver v0.34.2 h1:WStKftnGeoKP4AZRz/BaAAEJvYp4mlZGN0UCv+uvsqo= +k8s.io/apiextensions-apiserver v0.34.2/go.mod h1:398CJrsgXF1wytdaanynDpJ67zG4Xq7yj91GrmYN2SE= k8s.io/apimachinery v0.22.0/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apimachinery v0.34.0 h1:eR1WO5fo0HyoQZt1wdISpFDffnWOvFLOOeJ7MgIv4z0= -k8s.io/apimachinery v0.34.0/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8= +k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= k8s.io/client-go v0.22.0/go.mod h1:GUjIuXR5PiEv/RVK5OODUsm6eZk7wtSWZSaSJbpFdGg= -k8s.io/client-go v0.34.0 h1:YoWv5r7bsBfb0Hs2jh8SOvFbKzzxyNo0nSb0zC19KZo= -k8s.io/client-go v0.34.0/go.mod h1:ozgMnEKXkRjeMvBZdV1AijMHLTh3pbACPvK7zFR+QQY= +k8s.io/client-go v0.35.0 h1:IAW0ifFbfQQwQmga0UdoH0yvdqrbwMdq9vIFEhRpxBE= +k8s.io/client-go v0.35.0/go.mod h1:q2E5AAyqcbeLGPdoRB+Nxe3KYTfPce1Dnu1myQdqz9o= k8s.io/component-base v0.22.0/go.mod h1:SXj6Z+V6P6GsBhHZVbWCw9hFjUdUYnJerlhhPnYCBCg= -k8s.io/component-base v0.32.1 h1:/5IfJ0dHIKBWysGV0yKTFfacZ5yNV1sulPh3ilJjRZk= -k8s.io/component-base v0.32.1/go.mod h1:j1iMMHi/sqAHeG5z+O9BFNCF698a1u0186zkjMZQ28w= +k8s.io/component-base v0.34.2 h1:HQRqK9x2sSAsd8+R4xxRirlTjowsg6fWCPwWYeSvogQ= +k8s.io/component-base v0.34.2/go.mod h1:9xw2FHJavUHBFpiGkZoKuYZ5pdtLKe97DEByaA+hHbM= +k8s.io/component-helpers v0.35.0 h1:wcXv7HJRksgVjM4VlXJ1CNFBpyDHruRI99RrBtrJceA= +k8s.io/component-helpers v0.35.0/go.mod h1:ahX0m/LTYmu7fL3W8zYiIwnQ/5gT28Ex4o2pymF63Co= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e h1:iW9ChlU0cU16w8MpVYjXk12dqQ4BPFBEgif+ap7/hqQ= +k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/controller-runtime v0.22.4 h1:GEjV7KV3TY8e+tJ2LCTxUTanW4z/FmNB7l327UfMq9A= +sigs.k8s.io/controller-runtime v0.22.4/go.mod h1:+QX1XUpTXN4mLoblf4tqr5CQcyHPAki2HLXqQMY6vh8= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.1 h1:JrhdFMqOd/+3ByqlP2I45kTOZmTRLBUm5pvRjeheg7E= +sigs.k8s.io/structured-merge-diff/v6 v6.3.1/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/helper.mk b/helper.mk new file mode 100644 index 00000000..01633ce3 --- /dev/null +++ b/helper.mk @@ -0,0 +1,17 @@ +# Copyright © 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Dell Technologies, Dell and other trademarks are trademarks of Dell Inc. +# or its subsidiaries. Other trademarks may be trademarks of their respective +# owners. + +generate: + go generate ./cmd/csi-powerstore + go run core/semver/semver.go -f mk > semver.mk + +download-csm-common: + git clone --depth 1 git@github.com:CSM/csm.git temp-repo + cp temp-repo/config/csm-common.mk . + rm -rf temp-repo + +vendor: + GOPRIVATE=github.com go mod vendor diff --git a/images.mk b/images.mk new file mode 100644 index 00000000..5121dc17 --- /dev/null +++ b/images.mk @@ -0,0 +1,22 @@ +# Copyright © 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Dell Technologies, Dell and other trademarks are trademarks of Dell Inc. +# or its subsidiaries. Other trademarks may be trademarks of their respective +# owners. + +include overrides.mk +include helper.mk + +images: download-csm-common generate vendor + $(eval include csm-common.mk) + @echo "Base Images is set to: $(BASEIMAGE)" + @echo "Building: $(IMAGE_REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG)" + $(BUILDER) build --pull $(NOCACHE) -t "$(IMAGE_REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG)" --build-arg GOIMAGE=$(DEFAULT_GOIMAGE) --build-arg BASEIMAGE=$(CSM_BASEIMAGE) . + +images-no-cache: + @echo "Building with --no-cache ..." + @make images NOCACHE=--no-cache + +push: + @echo "Pushing: $(IMAGE_REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG)" + $(BUILDER) push "$(IMAGE_REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG)" diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..c36cc9ce --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,7 @@ +site_name: 'csi-powerstore' +site_description: 'csi-powerstore Documentation.' +docs_dir: docs +plugins: + - techdocs-core +theme: + name: material diff --git a/mocks/FcConnector.go b/mocks/FcConnector.go index d265aa4b..a2649926 100644 --- a/mocks/FcConnector.go +++ b/mocks/FcConnector.go @@ -60,6 +60,24 @@ func (_m *FcConnector) DisconnectVolumeByDeviceName(ctx context.Context, name st return r0 } +// DisconnectVolumeByWWN provides a mock function with given fields: ctx, wwn +func (_m *FcConnector) DisconnectVolumeByWWN(ctx context.Context, wwn string) error { + ret := _m.Called(ctx, wwn) + + if len(ret) == 0 { + panic("no return value specified for DisconnectVolumeByWWN") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { + r0 = rf(ctx, wwn) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // GetInitiatorPorts provides a mock function with given fields: ctx func (_m *FcConnector) GetInitiatorPorts(ctx context.Context) ([]string, error) { ret := _m.Called(ctx) diff --git a/mocks/NodeInterface.go b/mocks/NodeInterface.go index ed976ede..b6d7e6a3 100644 --- a/mocks/NodeInterface.go +++ b/mocks/NodeInterface.go @@ -29,9 +29,9 @@ import ( context "context" reflect "reflect" - csi "github.com/container-storage-interface/spec/lib/go/csi" array "github.com/dell/csi-powerstore/v2/pkg/array" fs "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + csi "github.com/container-storage-interface/spec/lib/go/csi" gomock "go.uber.org/mock/gomock" ) diff --git a/mocks/NodeLabelsModifier.go b/mocks/NodeLabelsModifier.go deleted file mode 100644 index e36ef977..00000000 --- a/mocks/NodeLabelsModifier.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - Copyright (c) 2025 Dell Inc, or its subsidiaries. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Code generated by mockery. DO NOT EDIT. - -package mocks - -import ( - context "context" - - mock "github.com/stretchr/testify/mock" -) - -// NodeLabelsModifierInterface is an autogenerated mock type for the NodeLabelsModifierInterface type -type NodeLabelsModifierInterface struct { - mock.Mock -} - -// AddNVMeLabels provides a mock function with given fields: ctx, kubeNodeName -func (_m *NodeLabelsModifierInterface) AddNVMeLabels(ctx context.Context, kubeNodeName string, labelKey string, labelValue []string) error { - ret := _m.Called(ctx, kubeNodeName, labelKey, labelValue) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, string, []string) error); ok { - return rf(ctx, kubeNodeName, labelKey, labelValue) - } - - if rf, ok := ret.Get(0).(func(context.Context, string, string, []string)); ok { - rf(ctx, kubeNodeName, labelKey, labelValue) - } - - if rf, ok := ret.Get(0).(func(context.Context, string, string, []string) error); ok { - r0 = rf(ctx, kubeNodeName, labelKey, labelValue) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// NewNodeLabelsModifierInterface creates a new instance of NodeLabelsModifierInterface. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewNodeLabelsModifierInterface(t interface { - mock.TestingT - Cleanup(func()) -}) *NodeLabelsModifierInterface { - mock := &NodeLabelsModifierInterface{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/mocks/NodeLabelsRetriever.go b/mocks/NodeLabelsRetriever.go deleted file mode 100644 index d16dad98..00000000 --- a/mocks/NodeLabelsRetriever.go +++ /dev/null @@ -1,178 +0,0 @@ -/* - Copyright (c) 2023-2025 Dell Inc, or its subsidiaries. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Code generated by mockery. DO NOT EDIT. - -package mocks - -import ( - context "context" - - kubernetes "k8s.io/client-go/kubernetes" - - mock "github.com/stretchr/testify/mock" - - rest "k8s.io/client-go/rest" -) - -// NodeLabelsRetrieverInterface is an autogenerated mock type for the NodeLabelsRetrieverInterface type -type NodeLabelsRetrieverInterface struct { - mock.Mock -} - -// BuildConfigFromFlags provides a mock function with given fields: masterURL, kubeconfig -func (_m *NodeLabelsRetrieverInterface) BuildConfigFromFlags(masterURL string, kubeconfig string) (*rest.Config, error) { - ret := _m.Called(masterURL, kubeconfig) - - var r0 *rest.Config - var r1 error - if rf, ok := ret.Get(0).(func(string, string) (*rest.Config, error)); ok { - return rf(masterURL, kubeconfig) - } - if rf, ok := ret.Get(0).(func(string, string) *rest.Config); ok { - r0 = rf(masterURL, kubeconfig) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*rest.Config) - } - } - - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(masterURL, kubeconfig) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetNVMeUUIDs provides a mock function with given fields: ctx -func (_m *NodeLabelsRetrieverInterface) GetNVMeUUIDs(ctx context.Context) (map[string]string, error) { - ret := _m.Called(ctx) - - var r0 map[string]string - var r1 error - if rf, ok := ret.Get(0).(func(context.Context) (map[string]string, error)); ok { - return rf(ctx) - } - if rf, ok := ret.Get(0).(func(context.Context) map[string]string); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]string) - } - } - - if rf, ok := ret.Get(1).(func(context.Context) error); ok { - r1 = rf(ctx) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetNodeLabels provides a mock function with given fields: ctx, kubeNodeName -func (_m *NodeLabelsRetrieverInterface) GetNodeLabels(ctx context.Context, kubeNodeName string) (map[string]string, error) { - ret := _m.Called(ctx, kubeNodeName) - - var r0 map[string]string - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string) (map[string]string, error)); ok { - return rf(ctx, kubeNodeName) - } - if rf, ok := ret.Get(0).(func(context.Context, string) map[string]string); ok { - r0 = rf(ctx, kubeNodeName) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]string) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = rf(ctx, kubeNodeName) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// InClusterConfig provides a mock function with given fields: -func (_m *NodeLabelsRetrieverInterface) InClusterConfig() (*rest.Config, error) { - ret := _m.Called() - - var r0 *rest.Config - var r1 error - if rf, ok := ret.Get(0).(func() (*rest.Config, error)); ok { - return rf() - } - if rf, ok := ret.Get(0).(func() *rest.Config); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*rest.Config) - } - } - - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// NewForConfig provides a mock function with given fields: config -func (_m *NodeLabelsRetrieverInterface) NewForConfig(config *rest.Config) (*kubernetes.Clientset, error) { - ret := _m.Called(config) - - var r0 *kubernetes.Clientset - var r1 error - if rf, ok := ret.Get(0).(func(*rest.Config) (*kubernetes.Clientset, error)); ok { - return rf(config) - } - if rf, ok := ret.Get(0).(func(*rest.Config) *kubernetes.Clientset); ok { - r0 = rf(config) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*kubernetes.Clientset) - } - } - - if rf, ok := ret.Get(1).(func(*rest.Config) error); ok { - r1 = rf(config) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// NewNodeLabelsRetrieverInterface creates a new instance of NodeLabelsRetrieverInterface. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewNodeLabelsRetrieverInterface(t interface { - mock.TestingT - Cleanup(func()) -}) *NodeLabelsRetrieverInterface { - mock := &NodeLabelsRetrieverInterface{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/mocks/NodeVolumePublisher.go b/mocks/NodeVolumePublisher.go index 2ffdeaea..3ac208b0 100644 --- a/mocks/NodeVolumePublisher.go +++ b/mocks/NodeVolumePublisher.go @@ -21,10 +21,9 @@ package mocks import ( context "context" - csi "github.com/container-storage-interface/spec/lib/go/csi" fs "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" - - logrus "github.com/sirupsen/logrus" + "github.com/dell/csmlog" + csi "github.com/container-storage-interface/spec/lib/go/csi" mock "github.com/stretchr/testify/mock" ) @@ -35,11 +34,11 @@ type NodeVolumePublisher struct { } // Publish provides a mock function with given fields: ctx, logFields, _a2, cap, isRO, targetPath, stagingPath -func (_m *NodeVolumePublisher) Publish(ctx context.Context, logFields logrus.Fields, _a2 fs.Interface, cap *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) { +func (_m *NodeVolumePublisher) Publish(ctx context.Context, logFields csmlog.Fields, _a2 fs.Interface, cap *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) { ret := _m.Called(ctx, logFields, _a2, cap, isRO, targetPath, stagingPath) var r0 *csi.NodePublishVolumeResponse - if rf, ok := ret.Get(0).(func(context.Context, logrus.Fields, fs.Interface, *csi.VolumeCapability, bool, string, string) *csi.NodePublishVolumeResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, csmlog.Fields, fs.Interface, *csi.VolumeCapability, bool, string, string) *csi.NodePublishVolumeResponse); ok { r0 = rf(ctx, logFields, _a2, cap, isRO, targetPath, stagingPath) } else { if ret.Get(0) != nil { @@ -48,7 +47,7 @@ func (_m *NodeVolumePublisher) Publish(ctx context.Context, logFields logrus.Fie } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, logrus.Fields, fs.Interface, *csi.VolumeCapability, bool, string, string) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, csmlog.Fields, fs.Interface, *csi.VolumeCapability, bool, string, string) error); ok { r1 = rf(ctx, logFields, _a2, cap, isRO, targetPath, stagingPath) } else { r1 = ret.Error(1) diff --git a/mocks/NodeVolumeStager.go b/mocks/NodeVolumeStager.go index 3ebb3d5e..c1304de6 100644 --- a/mocks/NodeVolumeStager.go +++ b/mocks/NodeVolumeStager.go @@ -21,10 +21,9 @@ package mocks import ( context "context" - csi "github.com/container-storage-interface/spec/lib/go/csi" fs "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" - - logrus "github.com/sirupsen/logrus" + "github.com/dell/csmlog" + csi "github.com/container-storage-interface/spec/lib/go/csi" mock "github.com/stretchr/testify/mock" ) @@ -35,11 +34,11 @@ type NodeVolumeStager struct { } // Stage provides a mock function with given fields: ctx, req, logFields, _a3, id -func (_m *NodeVolumeStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, logFields logrus.Fields, _a3 fs.Interface, id string) (*csi.NodeStageVolumeResponse, error) { +func (_m *NodeVolumeStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, logFields csmlog.Fields, _a3 fs.Interface, id string) (*csi.NodeStageVolumeResponse, error) { ret := _m.Called(ctx, req, logFields, _a3, id) var r0 *csi.NodeStageVolumeResponse - if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeStageVolumeRequest, logrus.Fields, fs.Interface, string) *csi.NodeStageVolumeResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeStageVolumeRequest, csmlog.Fields, fs.Interface, string) *csi.NodeStageVolumeResponse); ok { r0 = rf(ctx, req, logFields, _a3, id) } else { if ret.Get(0) != nil { @@ -48,7 +47,7 @@ func (_m *NodeVolumeStager) Stage(ctx context.Context, req *csi.NodeStageVolumeR } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *csi.NodeStageVolumeRequest, logrus.Fields, fs.Interface, string) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *csi.NodeStageVolumeRequest, csmlog.Fields, fs.Interface, string) error); ok { r1 = rf(ctx, req, logFields, _a3, id) } else { r1 = ret.Error(1) diff --git a/mocks/VolumeStager.go b/mocks/VolumeStager.go index f7993c30..7e0fc14e 100644 --- a/mocks/VolumeStager.go +++ b/mocks/VolumeStager.go @@ -5,10 +5,9 @@ package mocks import ( context "context" - csi "github.com/container-storage-interface/spec/lib/go/csi" fs "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" - - logrus "github.com/sirupsen/logrus" + "github.com/dell/csmlog" + csi "github.com/container-storage-interface/spec/lib/go/csi" mock "github.com/stretchr/testify/mock" ) @@ -19,7 +18,7 @@ type VolumeStager struct { } // Stage provides a mock function with given fields: ctx, req, logFields, _a3, id, isRemote -func (_m *VolumeStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, logFields logrus.Fields, _a3 fs.Interface, id string, isRemote bool) (*csi.NodeStageVolumeResponse, error) { +func (_m *VolumeStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, logFields csmlog.Fields, _a3 fs.Interface, id string, isRemote bool) (*csi.NodeStageVolumeResponse, error) { ret := _m.Called(ctx, req, logFields, _a3, id, isRemote) if len(ret) == 0 { @@ -28,10 +27,10 @@ func (_m *VolumeStager) Stage(ctx context.Context, req *csi.NodeStageVolumeReque var r0 *csi.NodeStageVolumeResponse var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeStageVolumeRequest, logrus.Fields, fs.Interface, string, bool) (*csi.NodeStageVolumeResponse, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeStageVolumeRequest, csmlog.Fields, fs.Interface, string, bool) (*csi.NodeStageVolumeResponse, error)); ok { return rf(ctx, req, logFields, _a3, id, isRemote) } - if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeStageVolumeRequest, logrus.Fields, fs.Interface, string, bool) *csi.NodeStageVolumeResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeStageVolumeRequest, csmlog.Fields, fs.Interface, string, bool) *csi.NodeStageVolumeResponse); ok { r0 = rf(ctx, req, logFields, _a3, id, isRemote) } else { if ret.Get(0) != nil { @@ -39,7 +38,7 @@ func (_m *VolumeStager) Stage(ctx context.Context, req *csi.NodeStageVolumeReque } } - if rf, ok := ret.Get(1).(func(context.Context, *csi.NodeStageVolumeRequest, logrus.Fields, fs.Interface, string, bool) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *csi.NodeStageVolumeRequest, csmlog.Fields, fs.Interface, string, bool) error); ok { r1 = rf(ctx, req, logFields, _a3, id, isRemote) } else { r1 = ret.Error(1) diff --git a/overrides.mk b/overrides.mk new file mode 100644 index 00000000..517d964f --- /dev/null +++ b/overrides.mk @@ -0,0 +1,29 @@ +# Copyright © 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Dell Technologies, Dell and other trademarks are trademarks of Dell Inc. +# or its subsidiaries. Other trademarks may be trademarks of their respective +# owners. + +IMAGE_REGISTRY?="sample_registry" +IMAGE_NAME="csi-powerstore" +IMAGE_TAG?=$(shell date +%Y%m%d%H%M%S) + +# figure out if podman or docker should be used (use podman if found) +ifneq (, $(shell which podman 2>/dev/null)) +export BUILDER=podman +else +export BUILDER=docker +endif + +# target to print some help regarding these overrides and how to use them +overrides-help: + @echo + @echo "The following environment variables can be set to control the build" + @echo + @echo "IMAGE_REGISTRY - The registry to push images to." + @echo " Current setting is: $(IMAGE_REGISTRY)" + @echo "IMAGE_NAME - The image name to be built." + @echo " Current setting is: $(IMAGE_NAME)" + @echo "IMAGE_TAG - The image tag to be built, default is the current date." + @echo " Current setting is: $(IMAGE_TAG)" + @echo diff --git a/pkg/array/array.go b/pkg/array/array.go index 1f896d10..47581dec 100644 --- a/pkg/array/array.go +++ b/pkg/array/array.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2024 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,16 +32,19 @@ import ( "sync" "time" - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/dell/csi-powerstore/v2/core" "github.com/dell/csi-powerstore/v2/pkg/identifiers" "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/k8sutils" + "github.com/dell/csm-dr/pkg/storage" + "github.com/dell/csmlog" csictx "github.com/dell/gocsi/context" "github.com/dell/gopowerstore" - log "github.com/sirupsen/logrus" + "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "gopkg.in/yaml.v3" + k8score "k8s.io/api/core/v1" + "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" + "sigs.k8s.io/yaml" ) var ( @@ -50,6 +53,7 @@ var ( ipToArrayMux sync.Mutex defaultMultiNasThreshold = 5 defaultMultiNasCooldown = 5 * time.Minute + log = csmlog.GetLogger() ) // Consumer provides methods for safe management of arrays @@ -69,6 +73,30 @@ type Locker struct { defaultArray *PowerStoreArray } +// Get returns a PowerStoreArray associated with the globalID +// if one exists in the Locker. +// +// Satisfies the ArrayLister interface for csm-dr +func (s *Locker) Get(globalID string) (storage.HealthChecker, error) { + log.Debugf("getting array for global ID %q", globalID) + return s.GetOneArray(globalID) +} + +// IsOnline returns true if the PowerStoreArray is online; false otherwise. +// Array status is determined by submitting a REST request for the array cluster +// using the gopowerstore API. +// +// Satisfies the HealthChecker interface for csm-dr. +func (psa *PowerStoreArray) IsOnline(ctx context.Context) bool { + log.Debugf("checking if array %q is online", psa.GlobalID) + _, err := psa.GetClient().GetCluster(ctx) + if err != nil { + log.Errorf("array %q is offline: %v", psa.GlobalID, err) + } + + return err == nil +} + // Arrays is a getter for list of arrays func (s *Locker) Arrays() map[string]*PowerStoreArray { s.arraysLock.Lock() @@ -240,23 +268,65 @@ func (n *NASCooldown) FallbackRetry(nasList []string) string { // It stores gopowerstore client that can be directly used to invoke PowerStore API calls. // This structure is supposed to be parsed from config and mainly is created by GetPowerStoreArrays function. type PowerStoreArray struct { - Endpoint string `yaml:"endpoint"` - GlobalID string `yaml:"globalID"` - Username string `yaml:"username"` - Password string `yaml:"password"` - NasName string `yaml:"nasName"` - BlockProtocol identifiers.TransportType `yaml:"blockProtocol"` - Insecure bool `yaml:"skipCertificateValidation"` - IsDefault bool `yaml:"isDefault"` - NfsAcls string `yaml:"nfsAcls"` - MetroTopology string `yaml:"metroTopology"` - Labels map[string]string `yaml:"labels"` + Endpoint string `yaml:"endpoint" json:"endpoint"` + GlobalID string `yaml:"globalID" json:"globalID"` + Username string `yaml:"username" json:"username"` + Password string `yaml:"password" json:"password"` + NasName string `yaml:"nasName" json:"nasName"` + BlockProtocol identifiers.TransportType `yaml:"blockProtocol" json:"blockProtocol"` + Insecure bool `yaml:"skipCertificateValidation" json:"skipCertificateValidation"` + IsDefault bool `yaml:"isDefault" json:"isDefault"` + NfsAcls string `yaml:"nfsAcls" json:"nfsAcls"` + + // MetroTopology describes the desired topology of the hosts configured for + // metro replication. Accepted values are "Uniform" and "Non-Uniform", with + // "Non-Uniform" not yet being implemented. + // + // Deprecated: MetroTopology is deprecated and remains only for purposes of + // backward compatibility. Use HostConnectivity instead. + MetroTopology string `yaml:"metroTopology" json:"metroTopology"` + // Labels is a set of labels, ANDed to build a node selector query. Used in + // conjunction with MetroTopology to identify nodes that should register + // metro hosts in this PowerStore system. + Labels map[string]string `yaml:"labels" json:"labels"` + + // HostConnectivity describes how nodes should register their hosts + // for this PowerStore system. + HostConnectivity *HostConnectivity `yaml:"hostConnectivity" json:"hostConnectivity"` Client gopowerstore.Client IP string NASCooldownTracker NASCooldownTracker } +// HostConnectivity provides two host connectivity options for nodes that will +// register hosts in a PowerStore system -- Local and Metro. +type HostConnectivity struct { + // Local contains a NodeSelector term used to identify nodes that are locally + // connected to the current PowerStore array and should register their hosts locally. + Local k8score.NodeSelector `yaml:"local" json:"local"` + // Metro contains more-specific options for how to register a host for + // use with uniform metro replication. For non-uniform metro replication, + // hosts should be registered as Local. + Metro MetroConnectivityOptions `yaml:"metro" json:"metro"` +} + +// MetroConnectivityOptions provides options for how a host should be registered +// to optimize the connection for uniform metro replication. +type MetroConnectivityOptions struct { + // ColocatedLocal contains a NodeSelector term used to identify nodes that are + // colocated with the current PowerStore array and should register a host with the system. + ColocatedLocal k8score.NodeSelector `yaml:"colocatedLocal" json:"colocatedLocal"` + // ColocatedRemote contains a NodeSelector term used to identify nodes that are + // NOT colocated with the current PowerStore array, but are instead colocated with the + // metro replication target and should still register a host with this system as a secondary path. + ColocatedRemote k8score.NodeSelector `yaml:"colocatedRemote" json:"colocatedRemote"` + // ColocatedBoth contains a NodeSelector term used to identify nodes that are + // colocated with both the current PowerStore array and the metro replication target PowerStore + // array and should register a host with the current system. + ColocatedBoth k8score.NodeSelector `yaml:"colocatedBoth" json:"colocatedBoth"` +} + // GetNasName is a getter that returns name of configured NAS func (psa *PowerStoreArray) GetNasName() string { return psa.NasName @@ -342,7 +412,7 @@ func GetPowerStoreArrays(fs fs.Interface, filePath string) (map[string]*PowerSto "unable to create PowerStore client: %s", err.Error()) } c.SetCustomHTTPHeaders(http.Header{ - "Application-Type": {fmt.Sprintf("%s/%s", identifiers.VerboseName, core.SemVer)}, + "Application-Type": {fmt.Sprintf("%s/%s", identifiers.VerboseName, identifiers.ManifestSemver)}, }) c.SetLogger(&identifiers.CustomLogger{}) @@ -453,6 +523,7 @@ func ParseVolumeID(ctx context.Context, volumeHandleRaw string, defaultArray *PowerStoreArray, /*legacy support*/ vc *csi.VolumeCapability, /*legacy support*/ ) (volumeHandle VolumeHandle, err error) { + log = log.WithContext(ctx) log.Debugf("ParseVolumeID: parsing volume handle %s", volumeHandleRaw) if volumeHandleRaw == "" { @@ -529,6 +600,12 @@ func ParseVolumeID(ctx context.Context, volumeHandleRaw string, return volumeHandle, nil } +// IsMetro determines if the volume handle belongs to a metro volume by checking +// if the RemoteUUID field is non-empty. +func (v *VolumeHandle) IsMetro() bool { + return v.RemoteUUID != "" && v.RemoteArrayGlobalID != "" +} + // GetVolumeUUIDPrefix extracts the prefix, if any exists, from a volume ID with a UUID format. // The prefix is assumed to be all characters preceding the volume UUID including separators/delimiters, // e.g. '-'. If no prefix is found, or the volume ID is not of the UUID format, the function returns an @@ -553,6 +630,7 @@ func GetVolumeUUIDPrefix(volumeID string) (prefix string) { // GetLeastUsedActiveNAS finds the active NAS with the least FS count func GetLeastUsedActiveNAS(ctx context.Context, arr *PowerStoreArray, nasServers []string) (string, error) { + log := log.WithContext(ctx) nasList, err := arr.Client.GetNASServers(ctx) if err != nil { log.Errorf("Failed to fetch NAS servers: %v", err) @@ -633,3 +711,63 @@ func GetNASInCooldown(arr *PowerStoreArray, nasServers []string) []string { } return nasInCooldown } + +// checkConnectivity checks if kubeNode matches metro selector. +func (psa *PowerStoreArray) CheckConnectivity(ctx context.Context, kubeNodeID string) bool { + var err error + + // Check for backward compatibility + if psa.HostConnectivity == nil { + log.Warnf("HostConnectivity is not defined in secret for array %s", psa.Endpoint) + return true + } + node, err := k8sutils.GetNodeByCSINodeID(ctx, identifiers.Name, kubeNodeID, identifiers.KeyNodeID) + if err != nil { + log.Errorf("GetNodeByCSINodeID error %s kubeNodeID %s", err, kubeNodeID) + return false + } + log.Debugf("checking if kubeNode %s matches metro selector. volume for metro array %+v", kubeNodeID, *psa.HostConnectivity) + if psa.DoesNodeMatchMetroSelectors(node) { + log.Debugf("selector matched.volume for uniform array %s, on kubeNodeID %s", psa.Endpoint, kubeNodeID) + return true + } + localSelector, localErr := nodeaffinity.NewNodeSelector(&psa.HostConnectivity.Local) + if localErr == nil && localSelector.Match(node) { // Check if node matches any of the local selectors in array(for non-unform array) + log.Debugf("Local selector matched for non-uniform array %s, on kubeNodeID %s", psa.Endpoint, kubeNodeID) + return true + } + + return false +} + +// DoesNodeMatchMetroSelectors checks if the given node matches any of the provided metro selectors in secret +func (psa *PowerStoreArray) DoesNodeMatchMetroSelectors(node *k8score.Node) bool { + local, errLocal := nodeaffinity.NewNodeSelector(&psa.HostConnectivity.Metro.ColocatedLocal) + if errLocal != nil { + log.Debugf("Error creating local node selector: %v", errLocal) + } + + if errLocal == nil && local != nil && local.Match(node) { + return true + } + + remote, errRemote := nodeaffinity.NewNodeSelector(&psa.HostConnectivity.Metro.ColocatedRemote) + if errRemote != nil { + log.Debugf("Error creating remote node selector: %v", errRemote) + } + + if errRemote == nil && remote != nil && remote.Match(node) { + return true + } + + colocate, errColocate := nodeaffinity.NewNodeSelector(&psa.HostConnectivity.Metro.ColocatedBoth) + if errColocate != nil { + log.Debugf("Error creating colocated node selector: %v", errColocate) + } + + if errColocate == nil && colocate != nil && colocate.Match(node) { + return true + } + log.Debug("Node does not match any metro selectors") + return false +} diff --git a/pkg/array/array_test.go b/pkg/array/array_test.go index b77177aa..dd650668 100644 --- a/pkg/array/array_test.go +++ b/pkg/array/array_test.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2024 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,23 +23,24 @@ import ( "errors" "net/http" "os" - "reflect" "testing" "time" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/mocks" "github.com/dell/csi-powerstore/v2/pkg/array" "github.com/dell/csi-powerstore/v2/pkg/identifiers" "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" - sharednfs "github.com/dell/csm-sharednfs/nfs" + "github.com/dell/csm-dr/pkg/storage" "github.com/dell/gofsutil" "github.com/dell/gopowerstore" "github.com/dell/gopowerstore/api" gopowerstoremock "github.com/dell/gopowerstore/mocks" + "github.com/container-storage-interface/spec/lib/go/csi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" + k8score "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( @@ -419,52 +420,6 @@ func TestParseVolumeID(t *testing.T) { assert.Equal(t, validRemoteGlobalID, id.RemoteArrayGlobalID) assert.Equal(t, scsi, id.Protocol) }) - - localVolUUID := "aaaaaaaa-0000-bbbb-1111-cccccccccccc" - powerstoreLocalSystemID := "PS000000000001" - SharedNFSVolumeID := sharednfs.CsiNfsPrefixDash + localVolUUID + "/" + powerstoreLocalSystemID + "/" + scsi - type args struct { - ctx context.Context - volumeHandle string - defaultArray *array.PowerStoreArray - vc *csi.VolumeCapability - } - tests := []struct { - name string - args args - want array.VolumeHandle - wantErr bool - }{ - { - name: "parse volume handle for a host-based nfs volume", - args: args{ - ctx: context.Background(), - volumeHandle: SharedNFSVolumeID, - defaultArray: nil, - vc: nil, - }, - want: array.VolumeHandle{ - LocalUUID: sharednfs.CsiNfsPrefixDash + localVolUUID, - LocalArrayGlobalID: powerstoreLocalSystemID, - RemoteUUID: "", - RemoteArrayGlobalID: "", - Protocol: scsi, - }, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := array.ParseVolumeID(tt.args.ctx, tt.args.volumeHandle, tt.args.defaultArray, tt.args.vc) - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("ParseVolumeID() got = %v, want %v", got, tt.want) - } - if (err != nil) != tt.wantErr { - t.Errorf("ParseVolumeID() error = %v, wantErr %v", err, tt.wantErr) - return - } - }) - } } func TestLocker_UpdateArrays(t *testing.T) { @@ -799,13 +754,6 @@ func Test_getVolumeIDPrefix(t *testing.T) { }, wantPrefix: "", }, - { - name: "volume UUID with host-based nfs prefix", - args: args{ - ID: sharednfs.CsiNfsPrefixDash + validBlockVolumeUUID, - }, - wantPrefix: sharednfs.CsiNfsPrefixDash, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -816,3 +764,421 @@ func Test_getVolumeIDPrefix(t *testing.T) { }) } } + +func TestLocker_Get(t *testing.T) { + tests := []struct { + name string // description of this test case + // Named input parameters for target function. + globalID string + locker func(tt *testing.T) *array.Locker + want storage.HealthChecker + wantErr bool + }{ + { + name: "fail", + globalID: "bad-id", + locker: func(_ *testing.T) *array.Locker { + return &array.Locker{} + }, + want: nil, + wantErr: true, + }, + { + name: "success", + globalID: validGlobalID, + locker: func(_ *testing.T) *array.Locker { + locker := array.Locker{} + locker.SetArrays(map[string]*array.PowerStoreArray{ + validGlobalID: { + GlobalID: validGlobalID, + }, + }) + return &locker + }, + want: &array.PowerStoreArray{}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := tt.locker(t) + got, gotErr := s.Get(tt.globalID) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("Get() failed: %v", gotErr) + } + return + } + if tt.wantErr { + t.Fatal("Get() succeeded unexpectedly") + } + + if !tt.wantErr && got == nil { + t.Errorf("Get() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPowerStoreArray_IsOnline(t *testing.T) { + tests := []struct { + name string // description of this test case + array func(tt *testing.T) *array.PowerStoreArray + want bool + }{ + { + name: "is online", + array: func(tt *testing.T) *array.PowerStoreArray { + mArray := gopowerstoremock.NewClient(tt) + mArray.On("GetCluster", mock.Anything).Return(gopowerstore.Cluster{}, nil) + + return &array.PowerStoreArray{ + Client: mArray, + } + }, + want: true, + }, + { + name: "is online", + array: func(tt *testing.T) *array.PowerStoreArray { + mArray := gopowerstoremock.NewClient(tt) + mArray.On("GetCluster", mock.Anything).Return(gopowerstore.Cluster{}, errors.New("failed to get cluster")) + + return &array.PowerStoreArray{ + Client: mArray, + } + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + psa := tt.array(t) + + got := psa.IsOnline(context.Background()) + + if got != tt.want { + t.Errorf("IsOnline() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestVolumeHandle_IsMetro(t *testing.T) { + tests := []struct { + name string + volumeHandle array.VolumeHandle + want bool + }{ + { + name: "is metro", + volumeHandle: array.VolumeHandle{ + LocalUUID: validBlockVolumeUUID, + LocalArrayGlobalID: validGlobalID, + Protocol: scsi, + RemoteUUID: validRemoteBlockVolumeUUID, + RemoteArrayGlobalID: validRemoteGlobalID, + }, + want: true, + }, + { + name: "is not metro", + volumeHandle: array.VolumeHandle{ + LocalUUID: validBlockVolumeUUID, + LocalArrayGlobalID: validGlobalID, + Protocol: scsi, + RemoteUUID: "", + RemoteArrayGlobalID: "", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v := tt.volumeHandle + + got := v.IsMetro() + + if got != tt.want { + t.Errorf("IsMetro() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestDoesNodeMatchMetroSelectors(t *testing.T) { + t.Run("Node matches local node selector", func(t *testing.T) { + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nodeid", + Labels: map[string]string{ + "zone1": "local", // does not match the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"nodeid"}`, + }, + }, + } + metroArr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "zone1", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"local"}, + }, + }, + }, + }, + }, + }, + }, + GlobalID: "APM0012345", + Endpoint: "https://192.168.0.3/api/rest", + Username: "admin", + Password: "pass", + Insecure: true, + IP: "192.168.0.3", + } + result := metroArr.DoesNodeMatchMetroSelectors(node) + assert.True(t, result) + }) + + t.Run("Node matches colocated node selector", func(t *testing.T) { + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nodeid", + Labels: map[string]string{ + "zone3": "both", // does not match the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"nodeid"}`, + }, + }, + } + metroArr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedBoth: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "zone3", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"both"}, + }, + }, + }, + }, + }, + }, + }, + GlobalID: "APM0012345", + Endpoint: "https://192.168.0.3/api/rest", + Username: "admin", + Password: "pass", + Insecure: true, + IP: "192.168.0.3", + } + result := metroArr.DoesNodeMatchMetroSelectors(node) + assert.True(t, result) + }) + + t.Run("Node matches remote node selector", func(t *testing.T) { + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nodeid", + Labels: map[string]string{ + "zone2": "remote", // does not match the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"nodeid"}`, + }, + }, + } + metroArr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "zone2", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"remote"}, + }, + }, + }, + }, + }, + }, + }, + GlobalID: "APM0012345", + Endpoint: "https://192.168.0.3/api/rest", + Username: "admin", + Password: "pass", + Insecure: true, + IP: "192.168.0.3", + } + result := metroArr.DoesNodeMatchMetroSelectors(node) + assert.True(t, result) + }) + + t.Run("Node does not match any node selector", func(t *testing.T) { + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nodeid", + Labels: map[string]string{ + "zone2": "local", // does not match the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"nodeid"}`, + }, + }, + } + metroArr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "zone2", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"remote"}, + }, + }, + }, + }, + }, + }, + }, + GlobalID: "APM0012345", + Endpoint: "https://192.168.0.3/api/rest", + Username: "admin", + Password: "pass", + Insecure: true, + IP: "192.168.0.3", + } + result := metroArr.DoesNodeMatchMetroSelectors(node) + assert.False(t, result) + }) + + t.Run("Error creating local node selector", func(t *testing.T) { + node := &k8score.Node{} + arr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"value"}, + }, + }, + }, + }, + }, + }, + }, + } + result := arr.DoesNodeMatchMetroSelectors(node) + assert.False(t, result) + }) + t.Run("Error creating remote node selector", func(t *testing.T) { + node := &k8score.Node{} + arr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"value"}, + }, + }, + }, + }, + }, + }, + }, + } + result := arr.DoesNodeMatchMetroSelectors(node) + assert.False(t, result) + }) + t.Run("Error creating Both node selector", func(t *testing.T) { + node := &k8score.Node{} + arr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedBoth: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"value"}, + }, + }, + }, + }, + }, + }, + }, + } + result := arr.DoesNodeMatchMetroSelectors(node) + assert.False(t, result) + }) + t.Run("Node does not find any metronode selector", func(t *testing.T) { + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nodeid", + Labels: map[string]string{ + "zone2": "local", // does not match the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"nodeid"}`, + }, + }, + } + metroArr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "zone4", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"local"}, + }, + }, + }, + }, + }, + }, + GlobalID: "APM0012345", + Endpoint: "https://192.168.0.3/api/rest", + Username: "admin", + Password: "pass", + Insecure: true, + IP: "192.168.0.3", + } + result := metroArr.DoesNodeMatchMetroSelectors(node) + assert.False(t, result) + }) +} diff --git a/pkg/array/metro_utils.go b/pkg/array/metro_utils.go new file mode 100644 index 00000000..4d09176b --- /dev/null +++ b/pkg/array/metro_utils.go @@ -0,0 +1,200 @@ +/* + * + * Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package array + +import ( + "context" + "time" + + drv1 "github.com/dell/csm-dr/api/v1" + drv1Client "github.com/dell/csm-dr/pkg/client" + "github.com/dell/gopowerstore" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrlClient "sigs.k8s.io/controller-runtime/pkg/client" +) + +var GetDRClientFunc = drv1Client.Get + +type MetroFracturedResponse struct { + IsFractured bool + VolumeName string + State string +} + +const ( + MediumTimeout = 30 * time.Second + MetroPrefixRegex = `^Metro_(Demote|Promote|Reprotect).*` +) + +func IsMetroFractured(ctx context.Context, client gopowerstore.Client, id string) (*MetroFracturedResponse, error) { + log := log.WithContext(ctx) + arrayVolume, err := client.GetVolume(ctx, id) + if err != nil { + return nil, err + } + + if arrayVolume.MetroReplicationSessionID != "" { + log.Infof("[METRO] MetroReplicationSessionID %s", arrayVolume.MetroReplicationSessionID) + + ctxLocal, cancelLocal := context.WithTimeout(ctx, 5*time.Second) + defer cancelLocal() + replicationSession, err := client.GetReplicationSessionByID(ctxLocal, arrayVolume.MetroReplicationSessionID) + if err != nil { + log.Errorf("[METRO] Unable to get replication session information by ID: %s, errror: %s", arrayVolume.MetroReplicationSessionID, err.Error()) + return nil, err + } + + if replicationSession.State == "Fractured" { + // We should only go here if the replicationSession is Fractured. + log.Infof("[METRO] ReplicationSession Status %s, LocalResourceState %s", replicationSession.State, replicationSession.LocalResourceState) + + return &MetroFracturedResponse{true, arrayVolume.Name, replicationSession.LocalResourceState}, nil + } + } + + return &MetroFracturedResponse{false, arrayVolume.Name, ""}, nil +} + +// checkMetroState checks metro state of a volume. +// Tries to get metroState from the localArray first. if there was error fetching this, tries to get the metro state from the remote array. +// Parameters: volumeHandle of the metro volume and the clients for the local and remote arrays. +// Returns: MetroFracturedResponse, bool indicating if localVolume of the metro was demoted or not and error +// - empty MetroFracturedResponse , false and error in case of error checking metro state. +// - MetroFracturedResponse(including isFractured and volumeName), true, nil error in case metro is Fractured and localVolume is demoted. +// - MetroFracturedResponse(including isFractured and volumeName), false, nil error in case metro is Fractured and localVolume is promoted. +// +// MetroFracturedResponse ( includes isFractured and volumeName which are used from the response) , a boolean that indicates whether the localVolume of the metro was demoted or not and error. +func CheckMetroState(ctx context.Context, volumeHandle VolumeHandle, localClient gopowerstore.Client, remoteClient gopowerstore.Client) (*MetroFracturedResponse, bool, error) { + log := log.WithContext(ctx) + localDemoted := false + ctxLocal, cancelLocal := context.WithTimeout(context.Background(), MediumTimeout) + defer cancelLocal() + metroResp, err := IsMetroFractured(ctxLocal, localClient, volumeHandle.LocalUUID) + if err != nil { + log.Errorf("error checking on local array if metro is fractured: %s", err.Error()) + ctxRemote, cancelRemote := context.WithTimeout(context.Background(), MediumTimeout) + defer cancelRemote() + metroResp, err = IsMetroFractured(ctxRemote, remoteClient, volumeHandle.RemoteUUID) + if err != nil { + log.Errorf("error checking on remote array if metro is fractured: %s", err.Error()) + return metroResp, false, err + } + + if metroResp.IsFractured && (metroResp.State == "Demoted" || metroResp.State == "System_Demoted") { + // Remote is Demoted. So local must be Promoted + localDemoted = false + } else { + localDemoted = true + } + } else { + if metroResp.IsFractured && (metroResp.State == "Demoted" || metroResp.State == "System_Demoted") { + localDemoted = true + } + } + return metroResp, localDemoted, nil +} + +func CreateOrUpdateJournalEntry(ctx context.Context, name string, + volumeHandle VolumeHandle, deferredArrayID, nodeName, operation string, + request []byte, +) error { + log := log.WithContext(ctx) + + id := volumeHandle.LocalUUID + arrayID := volumeHandle.LocalArrayGlobalID + remoteArrayID := volumeHandle.RemoteArrayGlobalID + + drClient, err := GetDRClientFunc(ctx) + if err != nil { + log.Errorf("[METRO] Unable to get dr client, error: %s", err.Error()) + return err + } + + var journal drv1.VolumeJournal + key := ctrlClient.ObjectKey{ + Name: "journal-" + name, + } + + deferEntry := drv1.JournalEntry{ + Operation: operation, + Status: "pending-reconciliation", + Time: time.Now().Format(time.RFC3339), + Host: nodeName, + Array: deferredArrayID, + Request: request, + } + + err = drClient.Get(ctx, key, &journal) + if err != nil { + if !k8sErrors.IsNotFound(err) { + log.Errorf("Unable to retrieve volume journal: %s", err.Error()) + return err + } + + // We didn't find the entry so we would need to create it. + journal = drv1.VolumeJournal{ + ObjectMeta: metav1.ObjectMeta{ + Name: "journal-" + name, + }, + Spec: drv1.VolumeJournalSpec{ + VolumeUUID: id, + OriginalArray: arrayID, + FailoverArray: remoteArrayID, + JournalEntries: []drv1.JournalEntry{ + deferEntry, + }, + }, + } + + err = drClient.Create(context.Background(), &journal) + if err != nil { + log.Errorf("[METRO] Error creating volume journals: %s", err.Error()) + return err + } + + log.Infof("[METRO] Successfully created volume journal: %s", journal.Name) + return nil + } + + found := false + for i, entry := range journal.Spec.JournalEntries { + if entry.Operation == operation { + if entry.Status == "pending-reconciliation" && entry.Host == nodeName { + journal.Spec.JournalEntries[i] = deferEntry + } + + found = true + + break + } + } + + if !found { + journal.Spec.JournalEntries = append(journal.Spec.JournalEntries, deferEntry) + } + + err = drClient.Update(ctx, &journal) + if err != nil { + log.Errorf("Unable to update volume journal: %s", err) + return err + } + + return nil +} diff --git a/pkg/array/metro_utils_test.go b/pkg/array/metro_utils_test.go new file mode 100644 index 00000000..143840c7 --- /dev/null +++ b/pkg/array/metro_utils_test.go @@ -0,0 +1,602 @@ +/* + * + * Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package array + +import ( + "context" + "errors" + "fmt" + "testing" + + drv1 "github.com/dell/csm-dr/api/v1" + "github.com/dell/gopowerstore" + gopowerstoremock "github.com/dell/gopowerstore/mocks" + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/gogo/protobuf/proto" + "github.com/google/uuid" + "github.com/stretchr/testify/mock" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestIsMetroFractured(t *testing.T) { + volumeHandle := VolumeHandle{ + LocalUUID: uuid.New().String(), + } + replicationSessionID := uuid.New().String() + + tests := []struct { + name string + client gopowerstore.Client + before func(*gopowerstoremock.Client) + wantResponse *MetroFracturedResponse + wantErr error + }{ + { + name: "IsMetroFractured - Not Replication Session ID", + client: new(gopowerstoremock.Client), + before: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{ + ID: uuid.New().String(), + Name: "myVolume", + MetroReplicationSessionID: "", + }, nil) + }, + wantResponse: &MetroFracturedResponse{IsFractured: false, VolumeName: "myVolume", State: ""}, + wantErr: nil, + }, + { + name: "IsMetroFractured - Replication Session is OK", + client: new(gopowerstoremock.Client), + before: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{ + ID: uuid.New().String(), + Name: "myVolume", + MetroReplicationSessionID: replicationSessionID, + }, nil) + + client.On("GetReplicationSessionByID", mock.Anything, mock.Anything).Return( + gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "OK", + }, nil) + }, + wantResponse: &MetroFracturedResponse{IsFractured: false, VolumeName: "myVolume", State: ""}, + wantErr: nil, + }, + { + name: "IsMetroFractured - Replication Session is Fractured", + client: new(gopowerstoremock.Client), + before: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{ + ID: uuid.New().String(), + Name: "myVolume", + MetroReplicationSessionID: replicationSessionID, + }, nil) + + client.On("GetReplicationSessionByID", mock.Anything, mock.Anything).Return( + gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + }, + wantResponse: &MetroFracturedResponse{IsFractured: true, VolumeName: "myVolume", State: "Promoted"}, + wantErr: nil, + }, + { + name: "IsMetroFractured - error: unable to get volume", + client: new(gopowerstoremock.Client), + before: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("unable to get volume")) + }, + wantResponse: nil, + wantErr: errors.New("unable to get volume"), + }, + { + name: "IsMetroFractured - error: unable to get replication session by ID", + client: new(gopowerstoremock.Client), + before: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{ + ID: uuid.New().String(), + Name: "myVolume", + MetroReplicationSessionID: replicationSessionID, + }, nil) + + client.On("GetReplicationSessionByID", mock.Anything, mock.Anything).Return( + gopowerstore.ReplicationSession{}, errors.New("unable to get replication session by ID")) + }, + wantResponse: nil, + wantErr: errors.New("unable to get replication session by ID"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.before != nil { + tt.before(tt.client.(*gopowerstoremock.Client)) + } + + response, err := IsMetroFractured(t.Context(), tt.client, volumeHandle.LocalUUID) + if err != nil { + if errors.Is(err, tt.wantErr) { + t.Errorf("IsMetroFractured() received unexpected error: got %v, want %v", err, tt.wantErr) + } + } + + if tt.wantResponse != nil { + if tt.wantResponse.IsFractured != response.IsFractured || tt.wantResponse.VolumeName != response.VolumeName || tt.wantResponse.State != response.State { + t.Errorf("IsMetroFractured() received unexpected response: got %v, want %v", response, tt.wantResponse) + } + } + }) + } +} + +func TestCheckMetroState(t *testing.T) { + tests := []struct { + name string + volumeHandle VolumeHandle + localClient gopowerstore.Client + remoteClient gopowerstore.Client + wantResponse *MetroFracturedResponse + wantLocalDemoted bool + wantErr error + beforeLocal func(*gopowerstoremock.Client) + beforeRemote func(*gopowerstoremock.Client) + }{ + { + name: "CheckMetroState - Local volume is demoted", + volumeHandle: VolumeHandle{ + LocalUUID: "local-uuid", + RemoteUUID: "remote-uuid", + }, + localClient: new(gopowerstoremock.Client), + remoteClient: new(gopowerstoremock.Client), + wantResponse: &MetroFracturedResponse{ + IsFractured: true, + VolumeName: "volume-name", + State: "Demoted", + }, + wantLocalDemoted: true, + wantErr: nil, + beforeLocal: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, "local-uuid").Return(gopowerstore.Volume{ + ID: "local-uuid", + Name: "volume-name", + MetroReplicationSessionID: "replication-session-id", + }, nil) + client.On("GetReplicationSessionByID", mock.Anything, "replication-session-id").Return(gopowerstore.ReplicationSession{ + ID: "replication-session-id", + State: "Fractured", + LocalResourceState: "Demoted", + }, nil) + }, + beforeRemote: func(_ *gopowerstoremock.Client) { + // Not expected to be called + }, + }, + { + name: "CheckMetroState - Local volume is promoted", + volumeHandle: VolumeHandle{ + LocalUUID: "local-uuid", + RemoteUUID: "remote-uuid", + }, + localClient: new(gopowerstoremock.Client), + remoteClient: new(gopowerstoremock.Client), + wantResponse: &MetroFracturedResponse{ + IsFractured: true, + VolumeName: "volume-name", + State: "Promoted", + }, + wantLocalDemoted: false, + wantErr: nil, + beforeLocal: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, "local-uuid").Return(gopowerstore.Volume{ + ID: "local-uuid", + Name: "volume-name", + MetroReplicationSessionID: "replication-session-id", + }, nil) + client.On("GetReplicationSessionByID", mock.Anything, "replication-session-id").Return(gopowerstore.ReplicationSession{ + ID: "replication-session-id", + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + }, + beforeRemote: func(_ *gopowerstoremock.Client) { + // Not expected to be called + }, + }, + { + name: "CheckMetroState - Error getting local volume, remote volume promoted", + volumeHandle: VolumeHandle{ + LocalUUID: "local-uuid", + RemoteUUID: "remote-uuid", + }, + localClient: new(gopowerstoremock.Client), + remoteClient: new(gopowerstoremock.Client), + wantResponse: &MetroFracturedResponse{ + IsFractured: true, + VolumeName: "volume-name", + State: "Promoted", + }, + wantLocalDemoted: true, + wantErr: nil, + beforeLocal: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, "local-uuid").Return(gopowerstore.Volume{}, fmt.Errorf("error getting local volume")) + }, + beforeRemote: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, "remote-uuid").Return(gopowerstore.Volume{ + ID: "remote-uuid", + Name: "volume-name", + MetroReplicationSessionID: "replication-session-id", + }, nil) + client.On("GetReplicationSessionByID", mock.Anything, "replication-session-id").Return(gopowerstore.ReplicationSession{ + ID: "replication-session-id", + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + }, + }, + { + name: "CheckMetroState - Error getting local volume, remote volume Demoted", + volumeHandle: VolumeHandle{ + LocalUUID: "local-uuid", + RemoteUUID: "remote-uuid", + }, + localClient: new(gopowerstoremock.Client), + remoteClient: new(gopowerstoremock.Client), + wantResponse: &MetroFracturedResponse{ + IsFractured: true, + VolumeName: "volume-name", + State: "Demoted", + }, + wantLocalDemoted: false, + wantErr: nil, + beforeLocal: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, "local-uuid").Return(gopowerstore.Volume{}, fmt.Errorf("error getting local volume")) + }, + beforeRemote: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, "remote-uuid").Return(gopowerstore.Volume{ + ID: "remote-uuid", + Name: "volume-name", + MetroReplicationSessionID: "replication-session-id", + }, nil) + client.On("GetReplicationSessionByID", mock.Anything, "replication-session-id").Return(gopowerstore.ReplicationSession{ + ID: "replication-session-id", + State: "Fractured", + LocalResourceState: "Demoted", + }, nil) + }, + }, + { + name: "CheckMetroState - Error getting local volume and remote volume", + volumeHandle: VolumeHandle{ + LocalUUID: "local-uuid", + RemoteUUID: "remote-uuid", + }, + localClient: new(gopowerstoremock.Client), + remoteClient: new(gopowerstoremock.Client), + wantResponse: nil, + wantLocalDemoted: false, + wantErr: fmt.Errorf("error getting remote volume"), + beforeLocal: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, "local-uuid").Return(gopowerstore.Volume{}, fmt.Errorf("error getting local volume")) + }, + beforeRemote: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, "remote-uuid").Return(gopowerstore.Volume{}, fmt.Errorf("error getting remote volume")) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.beforeLocal(tt.localClient.(*gopowerstoremock.Client)) + tt.beforeRemote(tt.remoteClient.(*gopowerstoremock.Client)) + + response, localDemoted, err := CheckMetroState(context.Background(), tt.volumeHandle, tt.localClient, tt.remoteClient) + + if tt.wantResponse != nil { + if tt.wantResponse.IsFractured != response.IsFractured || tt.wantResponse.VolumeName != response.VolumeName || tt.wantResponse.State != response.State { + t.Errorf("CheckMetroState() response = %v, want %v", response, tt.wantResponse) + } + } + + if localDemoted != tt.wantLocalDemoted { + t.Errorf("CheckMetroState() localDemoted = %v, want %v", localDemoted, tt.wantLocalDemoted) + } + + if (tt.wantErr != nil && err == nil) || (tt.wantErr == nil && err != nil) { + t.Errorf("CheckMetroState() error = %v, want %v", err, tt.wantErr) + } + }) + } +} + +func TestCreateOrUpdateJournalEntry(t *testing.T) { + defaultGetClientFunc := GetDRClientFunc + + volumeName := "my-volume" + volumeHandle := VolumeHandle{ + LocalUUID: uuid.New().String(), + } + nodeName := "myNode" + tests := []struct { + name string + operation string + wantErr error + init func(myClient client.Client) + before func(operation string) ([]byte, client.Client) + }{ + { + name: "CreateOrUpdateJournalEntry - Success: Creation of Journal", + operation: "NodeStageVolume", + init: func(myClient client.Client) { + GetDRClientFunc = func(_ context.Context) (client.Client, error) { + return myClient, nil + } + }, + before: func(_ string) ([]byte, client.Client) { + req := &csi.NodeStageVolumeRequest{ + VolumeId: uuid.NewString(), + } + + deferredRequest, err := proto.Marshal(req) + if err != nil { + return nil, nil + } + + volumeJournal := drv1.VolumeJournal{} + scheme := setupDrScheme() + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&volumeJournal).Build() + + return deferredRequest, client + }, + wantErr: nil, + }, + { + name: "CreateOrUpdateJournalEntry - Success: Update of Journal (found)", + operation: "NodeStageVolume", + init: func(myClient client.Client) { + GetDRClientFunc = func(_ context.Context) (client.Client, error) { + return myClient, nil + } + }, + before: func(operation string) ([]byte, client.Client) { + req := &csi.NodeStageVolumeRequest{ + VolumeId: uuid.NewString(), + } + + deferredRequest, err := proto.Marshal(req) + if err != nil { + return nil, nil + } + + volumeJournal := drv1.VolumeJournal{ + ObjectMeta: metav1.ObjectMeta{ + Name: "journal-" + volumeName, + }, + Spec: drv1.VolumeJournalSpec{ + JournalEntries: []drv1.JournalEntry{ + { + Operation: operation, + Status: "pending-reconciliation", + Host: nodeName, + }, + }, + }, + } + scheme := setupDrScheme() + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&volumeJournal).Build() + + return deferredRequest, client + }, + wantErr: nil, + }, + { + name: "CreateOrUpdateJournalEntry - Success: Update of Journal (not found)", + operation: "NodeStageVolume", + init: func(myClient client.Client) { + GetDRClientFunc = func(_ context.Context) (client.Client, error) { + return myClient, nil + } + }, + before: func(_ string) ([]byte, client.Client) { + req := &csi.NodeStageVolumeRequest{ + VolumeId: uuid.NewString(), + } + + deferredRequest, err := proto.Marshal(req) + if err != nil { + return nil, nil + } + + volumeJournal := drv1.VolumeJournal{ + ObjectMeta: metav1.ObjectMeta{ + Name: "journal-" + volumeName, + }, + Spec: drv1.VolumeJournalSpec{ + JournalEntries: []drv1.JournalEntry{ + { + Operation: "ControllerPublishVolume", + Status: "pending-reconciliation", + }, + }, + }, + } + scheme := setupDrScheme() + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&volumeJournal).Build() + + return deferredRequest, client + }, + wantErr: nil, + }, + { + name: "CreateOrUpdateJournalEntry - Error: Unable to get client", + operation: "NodeStageVolume", + init: func(_ client.Client) { + GetDRClientFunc = func(_ context.Context) (client.Client, error) { + return nil, errors.New("unable to get dr client") + } + }, + before: func(_ string) ([]byte, client.Client) { + volumeJournal := drv1.VolumeJournal{} + scheme := setupDrScheme() + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&volumeJournal).Build() + + return nil, client + }, + wantErr: errors.New("unable to get dr client"), + }, + { + name: "CreateOrUpdateJournalEntry - Error: CSMDR not registered", + operation: "NodeStageVolume", + init: func(myClient client.Client) { + GetDRClientFunc = func(_ context.Context) (client.Client, error) { + return myClient, nil + } + }, + before: func(_ string) ([]byte, client.Client) { + client := fake.NewClientBuilder().Build() + + return nil, client + }, + wantErr: errors.New("not registered"), + }, + { + name: "CreateOrUpdateJournalEntry - Error: Creation of Journal", + operation: "NodeStageVolume", + init: func(myClient client.Client) { + GetDRClientFunc = func(_ context.Context) (client.Client, error) { + return myClient, nil + } + }, + before: func(_ string) ([]byte, client.Client) { + req := &csi.NodeStageVolumeRequest{ + VolumeId: uuid.NewString(), + } + + deferredRequest, err := proto.Marshal(req) + if err != nil { + return nil, nil + } + + volumeJournal := drv1.VolumeJournal{} + scheme := setupDrScheme() + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&volumeJournal).Build() + + testClient := &failingClient{Client: client, failOperator: map[string]bool{"create": true}} + + return deferredRequest, testClient + }, + }, + { + name: "CreateOrUpdateJournalEntry - Error: Update of Journal", + operation: "NodeStageVolume", + init: func(myClient client.Client) { + GetDRClientFunc = func(_ context.Context) (client.Client, error) { + return myClient, nil + } + }, + before: func(operation string) ([]byte, client.Client) { + req := &csi.NodeStageVolumeRequest{ + VolumeId: uuid.NewString(), + } + + deferredRequest, err := proto.Marshal(req) + if err != nil { + return nil, nil + } + + volumeJournal := drv1.VolumeJournal{ + ObjectMeta: metav1.ObjectMeta{ + Name: "journal-" + volumeName, + }, + Spec: drv1.VolumeJournalSpec{ + JournalEntries: []drv1.JournalEntry{ + { + Operation: operation, + Status: "pending-reconciliation", + }, + }, + }, + } + scheme := setupDrScheme() + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&volumeJournal).Build() + + testClient := &failingClient{Client: client, failOperator: map[string]bool{"update": true}} + + return deferredRequest, testClient + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + request, client := tt.before(tt.operation) + t.Cleanup(func() { + GetDRClientFunc = defaultGetClientFunc + }) + + tt.init(client) + + err := CreateOrUpdateJournalEntry(t.Context(), volumeName, volumeHandle, volumeHandle.LocalArrayGlobalID, nodeName, tt.operation, request) + if err != nil { + if errors.Is(err, tt.wantErr) { + t.Errorf("IsMetroFractured() received unexpected error: got %v, want %v", err, tt.wantErr) + } + } + }) + } +} + +func setupDrScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + _ = drv1.AddToScheme(scheme) + return scheme +} + +// failingClient is a client that fails on create and update +type failingClient struct { + client.Client + failOperator map[string]bool +} + +func (f *failingClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if !f.failOperator["create"] { + return f.Client.Create(ctx, obj, opts...) + } + + return fmt.Errorf("simulated create failure") +} + +func (f *failingClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if !f.failOperator["update"] { + return f.Client.Update(ctx, obj, opts...) + } + + return fmt.Errorf("simulated create failure") +} diff --git a/pkg/controller/base.go b/pkg/controller/base.go index 7e9d2eb2..63169fbe 100644 --- a/pkg/controller/base.go +++ b/pkg/controller/base.go @@ -23,9 +23,9 @@ import ( "strings" "unicode/utf8" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/pkg/identifiers" "github.com/dell/gopowerstore" + "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/timestamppb" diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index fa570f97..dfe29e81 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2024 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,16 +23,17 @@ import ( "context" "errors" "fmt" + "regexp" "sort" "strconv" "strings" "sync" - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/dell/csi-powerstore/v2/core" "github.com/dell/csi-powerstore/v2/pkg/array" "github.com/dell/csi-powerstore/v2/pkg/identifiers" "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/k8sutils" + "github.com/dell/csmlog" commonext "github.com/dell/dell-csi-extensions/common" podmon "github.com/dell/dell-csi-extensions/podmon" csiext "github.com/dell/dell-csi-extensions/replication" @@ -40,7 +41,8 @@ import ( csictx "github.com/dell/gocsi/context" "github.com/dell/gopowerstore" "github.com/dell/gopowerstore/api" - log "github.com/sirupsen/logrus" + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/protobuf/proto" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -50,9 +52,10 @@ import ( // Interface provides most important controller methods. // This essentially serves as a wrapper for controller service that is used in ephemeral volumes. type Interface interface { - csi.ControllerServer - ProbeController(context.Context, *commonext.ProbeControllerRequest) (*commonext.ProbeControllerResponse, error) - RegisterAdditionalServers(*grpc.Server) + CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) + DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) + ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) + ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) array.Consumer } @@ -60,8 +63,9 @@ type Interface interface { type Service struct { Fs fs.Interface - externalAccess string - nfsAcls string + externalAccess string + exclusiveAccess bool + nfsAcls string array.Locker @@ -69,20 +73,45 @@ type Service struct { replicationPrefix string isHealthMonitorEnabled bool isAutoRoundOffFsSizeEnabled bool + IsCSMDREnabled bool } // maxVolumesSizeForArray - store the maxVolumesSizeForArray var maxVolumesSizeForArray = make(map[string]int64) +// function variables for mocking during testing of metro replication and deferrals related to metro replication +var ( + isNodeConnectedToArrayFunc = isNodeConnectedToArray + unpublishVolumeFunc = unpublishVolume + checkMetroStateFunc = array.CheckMetroState + createOrUpdateJournalEntryFunc = array.CreateOrUpdateJournalEntry +) + var mutex = &sync.Mutex{} +// Instantiate csmlog at package level +var log = csmlog.GetLogger() + // Init is a method that initializes internal variables of controller service func (s *Service) Init() error { ctx := context.Background() + kubeConfigPath, _ := csictx.LookupEnv(ctx, identifiers.EnvKubeConfigPath) + _, err := k8sutils.CreateKubeClientSet(kubeConfigPath) + if err != nil { + return fmt.Errorf("failed to create Kubernetes client: %s", err.Error()) + } + if nat, ok := csictx.LookupEnv(ctx, identifiers.EnvExternalAccess); ok { s.externalAccess = nat } + if exclusive, ok := csictx.LookupEnv(ctx, identifiers.EnvExclusiveAccess); ok { + // Only enable exclusive access if external access is configured + if s.externalAccess != "" { + s.exclusiveAccess = strings.EqualFold(exclusive, "true") + } + } + if replicationContextPrefix, ok := csictx.LookupEnv(ctx, identifiers.EnvReplicationContextPrefix); ok { s.replicationContextPrefix = replicationContextPrefix + "/" } @@ -112,6 +141,7 @@ func (s *Service) Init() error { // CreateVolume creates either FileSystem or Volume on storage array. func (s *Service) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + log := log.WithContext(ctx) params := req.GetParameters() // Get array from map @@ -251,7 +281,7 @@ func (s *Service) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest volumeSource := contentSource.GetVolume() if volumeSource != nil { - log.Printf("volume %s specified as volume content source", volumeSource.VolumeId) + log.Infof("volume %s specified as volume content source", volumeSource.VolumeId) volumeHandle, parseVolErr := array.ParseVolumeID(ctx, volumeSource.VolumeId, s.DefaultArray(), nil) if parseVolErr != nil { if apiError, ok := parseVolErr.(gopowerstore.APIError); ok && apiError.NotFound() { @@ -265,7 +295,7 @@ func (s *Service) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest } snapshotSource := contentSource.GetSnapshot() if snapshotSource != nil { - log.Printf("snapshot %s specified as volume content source", snapshotSource.SnapshotId) + log.Infof("snapshot %s specified as volume content source", snapshotSource.SnapshotId) volumeHandle, parseVolErr := array.ParseVolumeID(ctx, snapshotSource.SnapshotId, s.DefaultArray(), nil) if parseVolErr != nil { if apiError, ok := parseVolErr.(gopowerstore.APIError); ok && apiError.NotFound() { @@ -308,12 +338,10 @@ func (s *Service) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest var vg gopowerstore.VolumeGroup var remoteSystem gopowerstore.RemoteSystem var remoteSystemName string + var vgName string isMetroVolume := false // Check if replication is enabled if replicationEnabled == "true" { - if useNFS { - return nil, status.Error(codes.InvalidArgument, "replication not supported for NFS") - } log.Info("Preparing volume replication") @@ -363,62 +391,80 @@ func (s *Service) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest } } - vgName := vgPrefix + "-" + namespace + remoteSystemName + "-" + rpo + if useNFS { + vgName = vgPrefix + "-nfs-" + namespace + remoteSystemName + "-" + rpo + } else { + vgName = vgPrefix + "-" + namespace + remoteSystemName + "-" + rpo + } if len(vgName) > 128 { vgName = vgName[:128] } + if !useNFS { + vg, err = arr.Client.GetVolumeGroupByName(ctx, vgName) + if err != nil { + if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { + log.Infof("Volume group with name %s not found, creating it", vgName) - vg, err = arr.Client.GetVolumeGroupByName(ctx, vgName) - if err != nil { - if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { - log.Infof("Volume group with name %s not found, creating it", vgName) + // ensure protection policy exists + pp, err := EnsureProtectionPolicyExists(ctx, arr, vgName, remoteSystemName, rpoEnum) + if err != nil { + return nil, status.Errorf(codes.Internal, "can't ensure protection policy exists %s", err.Error()) + } - // ensure protection policy exists - pp, err := EnsureProtectionPolicyExists(ctx, arr, vgName, remoteSystemName, rpoEnum) - if err != nil { - return nil, status.Errorf(codes.Internal, "can't ensure protection policy exists %s", err.Error()) - } + group, err := arr.Client.CreateVolumeGroup(ctx, &gopowerstore.VolumeGroupCreate{ + Name: vgName, + ProtectionPolicyID: pp, + }) + if err != nil { + return nil, status.Errorf(codes.Internal, "can't create volume group: %s", err.Error()) + } - group, err := arr.Client.CreateVolumeGroup(ctx, &gopowerstore.VolumeGroupCreate{ - Name: vgName, - ProtectionPolicyID: pp, - }) - if err != nil { - return nil, status.Errorf(codes.Internal, "can't create volume group: %s", err.Error()) - } + vg, err = arr.Client.GetVolumeGroup(ctx, group.ID) + if err != nil { + return nil, status.Errorf(codes.Internal, "can't query volume group by id %s : %s", group.ID, err.Error()) + } - vg, err = arr.Client.GetVolumeGroup(ctx, group.ID) - if err != nil { - return nil, status.Errorf(codes.Internal, "can't query volume group by id %s : %s", group.ID, err.Error()) + } else { + return nil, status.Errorf(codes.Internal, "can't query volume group by name %s : %s", vgName, err.Error()) } - } else { - return nil, status.Errorf(codes.Internal, "can't query volume group by name %s : %s", vgName, err.Error()) + // if Replication mode is SYNC, check if the VolumeGroup is write-order consistent + if repMode == identifiers.SyncMode { + if !vg.IsWriteOrderConsistent { + return nil, status.Errorf(codes.Internal, "can't apply protection policy with sync rule if volume group is not write-order consistent") + } + } + // group exists, check that protection policy applied + if vg.ProtectionPolicyID == "" { + pp, err := EnsureProtectionPolicyExists(ctx, arr, vgName, remoteSystemName, rpoEnum) + if err != nil { + return nil, status.Errorf(codes.Internal, "can't ensure protection policy exists %s", err.Error()) + } + policyUpdate := gopowerstore.VolumeGroupChangePolicy{ProtectionPolicyID: pp} + _, err = arr.Client.UpdateVolumeGroupProtectionPolicy(ctx, vg.ID, &policyUpdate) + if err != nil { + return nil, status.Errorf(codes.Internal, "can't update volume group policy %s", err.Error()) + } + } + } + + // Pass the VolumeGroup to the creator so it can create the new volume inside the vg + if c, ok := creator.(*SCSICreator); ok { + c.vg = &vg } } else { - // if Replication mode is SYNC, check if the VolumeGroup is write-order consistent - if repMode == identifiers.SyncMode { - if !vg.IsWriteOrderConsistent { - return nil, status.Errorf(codes.Internal, "can't apply protection policy with sync rule if volume group is not write-order consistent") - } + pp, err := EnsureProtectionPolicyExists(ctx, arr, vgName, remoteSystemName, rpoEnum) + if err != nil { + return nil, status.Errorf(codes.Internal, "can't ensure protection policy exists %s", err.Error()) } - // group exists, check that protection policy applied - if vg.ProtectionPolicyID == "" { - pp, err := EnsureProtectionPolicyExists(ctx, arr, vgName, remoteSystemName, rpoEnum) - if err != nil { - return nil, status.Errorf(codes.Internal, "can't ensure protection policy exists %s", err.Error()) - } - policyUpdate := gopowerstore.VolumeGroupChangePolicy{ProtectionPolicyID: pp} - _, err = arr.Client.UpdateVolumeGroupProtectionPolicy(ctx, vg.ID, &policyUpdate) - if err != nil { - return nil, status.Errorf(codes.Internal, "can't update volume group policy %s", err.Error()) - } + log.Infof("Protection policy %s verified or created successfully.", pp) + err = arr.Client.ModifyNASByName(ctx, &gopowerstore.NASModify{ProtectionPolicyID: pp}, selectedNasName) + if err != nil { + return nil, status.Errorf(codes.Internal, "can't update NAS protection policy %s", err.Error()) } - } - // Pass the VolumeGroup to the creator so it can create the new volume inside the vg - if c, ok := creator.(*SCSICreator); ok { - c.vg = &vg + log.Infof("Protection policy %s applied to NAS server %s", pp, remoteSystemName) + } case identifiers.MetroMode: // handle Metro mode where metro is configured directly on the volume @@ -435,6 +481,7 @@ func (s *Service) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest default: return nil, status.Errorf(codes.InvalidArgument, "replication enabled but invalid replication mode specified in storage class") } + } params[identifiers.KeyVolumeDescription] = getDescription(req.GetParameters()) @@ -464,6 +511,11 @@ func (s *Service) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest } } + if isMetroVolume && !s.IsCSMDREnabled { + log.Info("Failed to create metro volume due to CSM DR not available.") + return nil, status.Error(codes.InvalidArgument, "Metro replication mode requires CSM-DR to be enabled") + } + if volumeResponse == nil { resp, createError := creator.Create(ctx, req, sizeInBytes, arr.GetClient()) if createError != nil { @@ -538,6 +590,7 @@ func (s *Service) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest // DeleteVolume deletes either FileSystem or Volume from storage array. func (s *Service) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { + log := log.WithContext(ctx) id := req.GetVolumeId() if id == "" { return nil, status.Error(codes.InvalidArgument, "volume ID is required") @@ -554,7 +607,6 @@ func (s *Service) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest id = volumeHandle.LocalUUID arrayID := volumeHandle.LocalArrayGlobalID protocol := volumeHandle.Protocol - remoteVolumeID := volumeHandle.RemoteUUID arr, ok := s.Arrays()[arrayID] if !ok { @@ -584,7 +636,7 @@ func (s *Service) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest if (len(nfsExportResp.RWRootHosts) == 1 || len(nfsExportResp.RWHosts) == 1) && s.externalAccess != "" { externalAccess, err := identifiers.ParseCIDR(s.externalAccess) if err != nil { - log.Debug("error occurred while parsing externalAccess: ", err.Error(), s.externalAccess) + log.Debugf("error occurred %s while parsing externalAccess: %s ", err.Error(), s.externalAccess) return nil, status.Errorf(codes.FailedPrecondition, "filesystem %s cannot be deleted as it has associated NFS or SMB shares.", id) @@ -594,12 +646,12 @@ func (s *Service) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest var modifyHostPayload gopowerstore.NFSExportModify // Removing externalAccess from RWHosts as well as RWRootHosts if len(nfsExportResp.RWRootHosts) == 1 && externalAccess == nfsExportResp.RWRootHosts[0] { - log.Debug("Trying to remove externalAccess IP with mask having RWRootHosts access while deleting the volume: ", externalAccess) + log.Debugf("Trying to remove externalAccess IP with mask having RWRootHosts access while deleting the volume: %s", externalAccess) modifyNFSExport = true modifyHostPayload.RemoveRWRootHosts = []string{externalAccess} } if len(nfsExportResp.RWHosts) == 1 && externalAccess == nfsExportResp.RWHosts[0] { - log.Debug("Trying to remove externalAccess IP with mask having RWHosts access while deleting the volume: ", externalAccess) + log.Debugf("Trying to remove externalAccess IP with mask having RWHosts access while deleting the volume: %s", externalAccess) modifyNFSExport = true modifyHostPayload.RemoveRWHosts = []string{externalAccess} } @@ -607,7 +659,7 @@ func (s *Service) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest if modifyNFSExport { _, err = arr.GetClient().ModifyNFSExport(ctx, &modifyHostPayload, nfsExportResp.ID) if err != nil { - log.Debug("failure when removing externalAccess from nfs export: ", err.Error()) + log.Debugf("failure when removing externalAccess from nfs export: %s", err.Error()) if apiError, ok := err.(gopowerstore.APIError); !(ok && apiError.HostAlreadyRemovedFromNFSExport()) { return nil, status.Errorf(codes.FailedPrecondition, "filesystem %s cannot be deleted as it has associated NFS or SMB shares.", @@ -638,85 +690,157 @@ func (s *Service) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest return nil, err } else if protocol == "scsi" { - vgs, err := arr.GetClient().GetVolumeGroupsByVolumeID(ctx, id) - if err != nil { - if apiError, ok := err.(gopowerstore.APIError); !ok || !apiError.NotFound() { - return nil, err + localDeleted := false + remoteDeleted := false + + var metroResp *array.MetroFracturedResponse + var remoteArray *array.PowerStoreArray + remoteVolumeID := volumeHandle.RemoteUUID + remoteArrayID := volumeHandle.RemoteArrayGlobalID + if volumeHandle.IsMetro() { + if arr, ok := s.Arrays()[remoteArrayID]; ok { + remoteArray = arr + } else { + return nil, status.Errorf(codes.InvalidArgument, "failed to find remote array with ID %s", remoteArrayID) } - } - - if len(vgs.VolumeGroup) != 0 { - // Remove volume from volume group - // TODO: If volume has multiple volume group then how we should find ours? - // TODO: Maybe adding volumegroup id/name to volume id can help? - _, err := arr.GetClient().RemoveMembersFromVolumeGroup(ctx, &gopowerstore.VolumeGroupMembers{VolumeIDs: []string{id}}, vgs.VolumeGroup[0].ID) + metroResp, _, err = array.CheckMetroState(ctx, volumeHandle, arr.GetClient(), remoteArray.GetClient()) if err != nil { - if apiError, ok := err.(gopowerstore.APIError); ok && apiError.VolumeAlreadyRemovedFromVolumeGroup() { // idempotency check - log.Debugf("Volume %s has already been removed from volume group %s", id, vgs.VolumeGroup[0].ID) // continue to delete volume - } else { - return nil, status.Errorf(codes.Internal, "failed to remove volume %s from volume group: %s", id, err.Error()) + if err, ok := err.(gopowerstore.APIError); ok && err.NotFound() { + // both local and remote attempts to get the metro state failed with NotFound errors + // indicating the volume is already deleted + return &csi.DeleteVolumeResponse{}, nil } + return nil, err + } + if metroResp.IsFractured { + log.Warnf("[METRO] metro volume %s is in a fractured state", req.GetVolumeId()) } + } - // Unassign protection policy - _, err = arr.GetClient().ModifyVolume(ctx, &gopowerstore.VolumeModify{ProtectionPolicyID: ""}, id) - if err != nil { - return nil, err + // Delete local volume + err = deleteISCSIVolume(ctx, volumeHandle, arr, id) + if err == nil { + localDeleted = true + } else { + log.Errorf("failed to delete volume %s: %v", id, err) + } + + // non-metro or not fractured just return results + if !volumeHandle.IsMetro() || !metroResp.IsFractured { + if localDeleted { + return &csi.DeleteVolumeResponse{}, nil } + return nil, err } - // TODO: if len(vgs.VolumeGroup == 1) && it is the last volume : delete volume group - // TODO: What to do with RPO snaps? - listSnaps, err := arr.GetClient().GetSnapshotsByVolumeID(ctx, id) - if err != nil { - return nil, status.Errorf(codes.Unknown, "failure getting snapshot: %s", err.Error()) + + // from here is metro and fractured array + err = deleteISCSIVolume(ctx, volumeHandle, remoteArray, remoteVolumeID) + if err == nil { + remoteDeleted = true + } else { + log.Errorf("failed to delete remote volume %s: %v", id, err) } - if len(listSnaps) > 0 { - return nil, status.Errorf(codes.FailedPrecondition, - "unable to delete volume -- %d snapshots based on this volume still exist.", len(listSnaps)) + + if localDeleted && remoteDeleted { + return &csi.DeleteVolumeResponse{}, nil + } + // return error if anything not deleted and k8s will retry + return nil, err + } + + return nil, status.Errorf(codes.InvalidArgument, "can't figure out protocol") +} + +func deleteISCSIVolume(ctx context.Context, _ array.VolumeHandle, arr *array.PowerStoreArray, id string) error { + log := log.WithContext(ctx) + vgs, err := arr.GetClient().GetVolumeGroupsByVolumeID(ctx, id) + if err != nil { + if apiError, ok := err.(gopowerstore.APIError); !ok || !apiError.NotFound() { + return err } + } - // Check if volume has metro session and end it - volume, err := arr.GetClient().GetVolume(ctx, id) + if len(vgs.VolumeGroup) != 0 { + // Remove volume from volume group + // TODO: If volume has multiple volume group then how we should find ours? + // TODO: Maybe adding volumegroup id/name to volume id can help? + _, err := arr.GetClient().RemoveMembersFromVolumeGroup(ctx, &gopowerstore.VolumeGroupMembers{VolumeIDs: []string{id}}, vgs.VolumeGroup[0].ID) if err != nil { - if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { - log.Infof("Volume %s not found, it may have been deleted.", id) - return &csi.DeleteVolumeResponse{}, nil + if apiError, ok := err.(gopowerstore.APIError); ok && apiError.VolumeAlreadyRemovedFromVolumeGroup() { // idempotency check + log.Debugf("Volume %s has already been removed from volume group %s", id, vgs.VolumeGroup[0].ID) // continue to delete volume + } else { + return status.Errorf(codes.Internal, "failed to remove volume %s from volume group: %s", id, err.Error()) } - return nil, status.Errorf(codes.Internal, "failure getting volume: %s", err.Error()) } - if volume.MetroReplicationSessionID != "" { - _, err = arr.GetClient().EndMetroVolume(ctx, id, &gopowerstore.EndMetroVolumeOptions{ - DeleteRemoteVolume: true, // delete remote volume when deleting local volume - }) - if err != nil { - return nil, status.Errorf(codes.Internal, "failure ending metro session on volume: %s", err.Error()) - } - } else if remoteVolumeID != "" { - log.Debugf("Expected metro session for volume %s, but it seems to have been already removed.", id) + + // Unassign protection policy + _, err = arr.GetClient().ModifyVolume(ctx, &gopowerstore.VolumeModify{ProtectionPolicyID: ""}, id) + if err != nil { + return err } + } - // Delete volume - _, err = arr.GetClient().DeleteVolume(ctx, nil, id) - if err == nil { - return &csi.DeleteVolumeResponse{}, nil + volume, err := arr.GetClient().GetVolume(ctx, id) + if err != nil { + if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { + log.Infof("Volume %s not found, it may have been deleted.", id) + return nil + } + return status.Errorf(codes.Internal, "failure getting volume: %s", err.Error()) + } + + // TODO: if len(vgs.VolumeGroup == 1) && it is the last volume : delete volume group + // TODO: What to do with RPO snaps? + listSnaps, err := arr.GetClient().GetSnapshotsByVolumeID(ctx, id) + if err != nil { + if apiError, ok := err.(gopowerstore.APIError); !ok || !apiError.NotFound() { + return status.Errorf(codes.Unknown, "failure getting snapshot: %s", err.Error()) + } + } + + blockingDeleteSnapshotCount := 0 + for _, snap := range listSnaps { + // Note: There is no other way to check if it is a metro user snapshot from the response. + regex := regexp.MustCompile(array.MetroPrefixRegex + volume.Name) + if !regex.MatchString(snap.Name) { + log.Warnf("Snapshot detected for metro volume %s, delete this to finish cleanup: %s", volume.Name, snap.Name) + blockingDeleteSnapshotCount++ + } + } + + if blockingDeleteSnapshotCount > 0 { + return status.Errorf(codes.FailedPrecondition, + "unable to delete volume %s -- %d snapshots based on this volume still exist.", volume.Name, blockingDeleteSnapshotCount) + } + + // Check if volume has metro session and end it + if volume.MetroReplicationSessionID != "" { + _, err = arr.GetClient().EndMetroVolume(ctx, id, &gopowerstore.EndMetroVolumeOptions{ + DeleteRemoteVolume: true, // delete remote volume when deleting local volume + }) + if err != nil { + return status.Errorf(codes.Internal, "failure ending metro session on volume: %s", err.Error()) } + } + + _, err = arr.GetClient().DeleteVolume(ctx, nil, id) + if err != nil { if apiError, ok := err.(gopowerstore.APIError); ok { if apiError.NotFound() { - return &csi.DeleteVolumeResponse{}, nil + return nil } if apiError.VolumeAttachedToHost() { - return nil, status.Errorf(codes.Internal, + return status.Errorf(codes.Internal, "volume with ID '%s' is still attached to host: %s", id, apiError.Error()) } } - return nil, err } - - return nil, status.Errorf(codes.InvalidArgument, "can't figure out protocol") + return err } // ControllerPublishVolume prepares Volume/FileSystem to be consumed by node by attaching/allowing access to the host. func (s *Service) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { + log := log.WithContext(ctx) id := req.GetVolumeId() kubeNodeID := req.GetNodeId() @@ -730,7 +854,7 @@ func (s *Service) ControllerPublishVolume(ctx context.Context, req *csi.Controll volumeHandle, err := array.ParseVolumeID(ctx, id, s.DefaultArray(), req.VolumeCapability) if err != nil { - log.Error(err) + log.Error(err.Error()) return nil, err } @@ -744,8 +868,8 @@ func (s *Service) ControllerPublishVolume(ctx context.Context, req *csi.Controll if !ok { return nil, status.Errorf(codes.InvalidArgument, "failed to find array with ID %s", arrayID) } - var remoteArray *array.PowerStoreArray - if remoteArrayID != "" { + remoteArray := &array.PowerStoreArray{} + if volumeHandle.IsMetro() { remoteArray, ok = s.Arrays()[remoteArrayID] if !ok { return nil, status.Errorf(codes.InvalidArgument, "failed to find remote array with ID %s", remoteArrayID) @@ -768,31 +892,123 @@ func (s *Service) ControllerPublishVolume(ctx context.Context, req *csi.Controll var publisher VolumePublisher if protocol == "nfs" { publisher = &NfsPublisher{ - ExternalAccess: s.externalAccess, + ExternalAccess: s.externalAccess, + ExclusiveAccess: s.exclusiveAccess, } } else { publisher = &SCSIPublisher{} } - if err := publisher.CheckIfVolumeExists(ctx, arr.GetClient(), id); err != nil { - return nil, err + localDemoted := false + var metroResp *array.MetroFracturedResponse + isMetroFractured := false + + if remoteVolumeID != "" { + ctxLocal, cancelLocal := context.WithTimeout(context.Background(), array.MediumTimeout) + defer cancelLocal() + metroResp, localDemoted, err = array.CheckMetroState(ctxLocal, volumeHandle, arr.GetClient(), remoteArray.GetClient()) + if err != nil { + return nil, err + } + isMetroFractured = metroResp.IsFractured + if isMetroFractured { + log.Warnf("[METRO] metro volume %s is in a fractured state", req.GetVolumeId()) + } + if localDemoted { + log.Warnf("[METRO] metro volume %s has been demoted", req.GetVolumeId()) + } + } else { + if err := publisher.CheckIfVolumeExists(ctx, arr.GetClient(), id); err != nil { + return nil, err + } } publishContext := make(map[string]string) - publishVolumeResponse, err := publisher.Publish(ctx, publishContext, req, arr.GetClient(), kubeNodeID, id, false) - if err != nil { - return nil, err + publishVolumeResponse := &csi.ControllerPublishVolumeResponse{} + localPublished, remotePublished := false, false + + hostRegisteredLocalArray := arr.CheckConnectivity(ctx, kubeNodeID) + if hostRegisteredLocalArray { + log.Infof("Volume is being published on node %s for array %s", kubeNodeID, arr.Endpoint) + ctxLocal, cancelLocal := context.WithTimeout(context.Background(), array.MediumTimeout) + defer cancelLocal() + publishReponse, publishErr := publisher.Publish(ctxLocal, publishContext, req, arr.GetClient(), kubeNodeID, id, false) + if publishErr != nil { + if isMetroFractured && localDemoted { + log.Infof("[METRO] Could not publish volume %s on node %s for array %s due to Metro Session Fracture", id, kubeNodeID, arr.Endpoint) + } else { + log.Errorf("Failed to publish volume %s on node %s for array %s: %s", id, kubeNodeID, arr.Endpoint, publishErr) + return nil, publishErr + } + } else { + log.Infof("Local volume %s published", id) + publishVolumeResponse = publishReponse + localPublished = true + } + } else { + log.Infof("skipping volume publish on node %s for array %s, topology does not match", kubeNodeID, arr.Endpoint) + } + + hostRegisteredRemoteArray := false + if volumeHandle.IsMetro() { + if hostRegisteredRemoteArray = remoteArray.CheckConnectivity(ctx, kubeNodeID); hostRegisteredRemoteArray { + log.Infof("Volume is being published on node %s for remote array %s", kubeNodeID, remoteArray.Endpoint) + ctxRemote, cancelRemote := context.WithTimeout(context.Background(), array.MediumTimeout) + defer cancelRemote() + publishReponse, publishErr := publisher.Publish(ctxRemote, publishContext, req, remoteArray.GetClient(), kubeNodeID, remoteVolumeID, true) + if publishErr != nil { + if isMetroFractured && !localDemoted { + // localDemoted == false implies local is Promoted and Remote is Demoted. + log.Infof("[METRO] Could not publish volume %s on node %s for array %s due to Metro Session Fracture", remoteVolumeID, kubeNodeID, remoteArray.Endpoint) + } else { + // remote is Promoted + log.Errorf("Failed to publish volume %s on node %s for array %s: %s", id, kubeNodeID, remoteArray.Endpoint, publishErr) + return nil, publishErr + } + } else { + log.Infof("Remote volume %s published", remoteVolumeID) + remotePublished = true + publishVolumeResponse = publishReponse + } + } else { + log.Debugf("skipping volume publish on node %s for remote array %s, topology does not match", kubeNodeID, remoteArray.Endpoint) + } + } + + // at least one publish should succeed for non-metro, non-uniform metro, and uniform metro + // if a publish fails for uniform metro, the failed request will be deferred by adding to the volume journal + if !localPublished && !remotePublished { + return nil, status.Error(codes.Internal, "failed to publish volume") } - if remoteArrayID != "" && remoteVolumeID != "" { // For Remote Metro volume - publishVolumeResponse, err = publisher.Publish(ctx, publishContext, req, remoteArray.GetClient(), kubeNodeID, remoteVolumeID, true) + // Handling Uniform Metro Fracture cases. Other failures would have returned error before this point. + // Deferred operations are relevant only for Uniform metro in a fractured state and when only one side of metro volume was published + // and other side was not. + // Uniform metro is confirmed by a metro volume handle, and host connectivity from the kubeNodeID to both arrays. + if volumeHandle.IsMetro() && hostRegisteredLocalArray && hostRegisteredRemoteArray { + if (localPublished && !remotePublished) || (!localPublished && remotePublished) { + deferredRequest, err := proto.Marshal(req) + if err != nil { + log.Errorf("[METRO] Error marshalling req: %s", err.Error()) + } + deferredArrayID := arrayID + if !remotePublished { + deferredArrayID = remoteArrayID + } + + err = createOrUpdateJournalEntryFunc(ctx, metroResp.VolumeName, volumeHandle, deferredArrayID, kubeNodeID, "ControllerPublishVolume", deferredRequest) + if err != nil { + log.Errorf("Could not create journal entry for operation %s for volume %s node %s array %s", "ControllerPublishVolume", id, kubeNodeID, arr.Endpoint) + } + } } - return publishVolumeResponse, err + return publishVolumeResponse, nil } // ControllerUnpublishVolume prepares Volume/FileSystem to be deleted by unattaching/disabling access to the host. func (s *Service) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { + log := log.WithContext(ctx) id := req.GetVolumeId() if id == "" { return nil, status.Error(codes.InvalidArgument, "volume ID is required") @@ -802,20 +1018,12 @@ func (s *Service) ControllerUnpublishVolume(ctx context.Context, req *csi.Contro if kubeNodeID == "" { return nil, status.Error(codes.InvalidArgument, "node ID is required") } - volumeHandle, err := array.ParseVolumeID(ctx, id, s.DefaultArray(), nil) if err != nil { - if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { - return &csi.ControllerUnpublishVolumeResponse{}, nil - } - return nil, status.Errorf(codes.Unknown, - "failure checking volume status for volume unpublishing: %s", err.Error()) + log.Error(err.Error()) + return nil, err } - - id = volumeHandle.LocalUUID arrayID := volumeHandle.LocalArrayGlobalID - protocol := volumeHandle.Protocol - remoteVolumeID := volumeHandle.RemoteUUID remoteArrayID := volumeHandle.RemoteArrayGlobalID arr, ok := s.Arrays()[arrayID] @@ -823,40 +1031,179 @@ func (s *Service) ControllerUnpublishVolume(ctx context.Context, req *csi.Contro return nil, status.Errorf(codes.InvalidArgument, "cannot find array %s", arrayID) } var remoteArray *array.PowerStoreArray - if remoteArrayID != "" { + isMetroFractured := false + localDemoted := false + metroResp := &array.MetroFracturedResponse{ + IsFractured: false, + } + + if volumeHandle.IsMetro() { remoteArray, ok = s.Arrays()[remoteArrayID] if !ok { return nil, status.Errorf(codes.InvalidArgument, "cannot find remote array %s", remoteArrayID) } - } - if protocol == "scsi" { - node, err := arr.GetClient().GetHostByName(ctx, kubeNodeID) + metroResp, localDemoted, err = checkMetroStateFunc(ctx, volumeHandle, arr.GetClient(), remoteArray.GetClient()) if err != nil { - if apiError, ok := err.(gopowerstore.APIError); ok && apiError.HostIsNotExist() { - // We need additional check here since we can just have host without ip in it - ipList := identifiers.GetIPListFromString(kubeNodeID) - if ipList == nil { - return nil, errors.New("can't find IP in nodeID") + if apiError, ok := err.(gopowerstore.APIError); ok { + if !apiError.NotFound() { + return nil, err } - ip := ipList[len(ipList)-1] - nodeID := kubeNodeID[:len(kubeNodeID)-len(ip)-1] - node, err = arr.GetClient().GetHostByName(ctx, nodeID) - if err != nil { - return nil, status.Errorf(codes.NotFound, "host with k8s node ID '%s' not found", kubeNodeID) + + // Not found due to potentially deleted volume through UI. Still need to unpublish. + log.Infof("[Metro] Volume with ID %s not found", id) + } + } + + if metroResp != nil { + isMetroFractured = metroResp.IsFractured + } + + if isMetroFractured { + log.Warnf("[METRO] metro volume %s is in a fractured state", req.GetVolumeId()) + } + + if localDemoted { + log.Warnf("[METRO] metro volume %s has been demoted", req.GetVolumeId()) + } + } + + localVolumeUnpublished := false + remoteVolumeUnpublished := false + response := &csi.ControllerUnpublishVolumeResponse{} + + // Check if it is Metro volume and with newer secret configuation + nodeConnectedToLocalArray := isNodeConnectedToArrayFunc(ctx, kubeNodeID, arr) + if nodeConnectedToLocalArray { + log.Debugf("Volume is being unpublished on node %s for array %s", kubeNodeID, arr.Endpoint) + ctxLocal, cancelLocal := context.WithTimeout(context.Background(), array.MediumTimeout) + defer cancelLocal() + resp, unpublishErr := unpublishVolumeFunc(ctxLocal, kubeNodeID, arr, &volumeHandle, nil) + if unpublishErr != nil { + if isMetroFractured && localDemoted { + // expected failure if Metro is Fractured and local array is down + log.Infof("[METRO] Could not unpublish volume %s on node %s for array %s due to Metro Session Fracture", id, kubeNodeID, arr.Endpoint) + } else { + log.Errorf("Failed to unpublish volume %s for array %s: %s", id, arr.Endpoint, err) + return nil, unpublishErr + } + } else { + log.Infof("Unpublished volume %s for array %s", id, arr.Endpoint) + localVolumeUnpublished = true + response = resp + } + } + nodeConnectedToRemoteArray := false + if volumeHandle.IsMetro() { + nodeConnectedToRemoteArray = isNodeConnectedToArrayFunc(ctx, kubeNodeID, remoteArray) + if nodeConnectedToRemoteArray { + log.Debugf("Volume is being unpublished on node %s for remote array %s", kubeNodeID, remoteArray.Endpoint) + ctxRemote, cancelRemote := context.WithTimeout(context.Background(), array.MediumTimeout) + defer cancelRemote() + resp, unpublishErr := unpublishVolumeFunc(ctxRemote, kubeNodeID, nil, &volumeHandle, remoteArray) + if unpublishErr != nil { + if isMetroFractured && !localDemoted { + // expected failure if Metro is Fractured and remote array is down + log.Infof("[METRO] Could not unpublish volume %s on node %s for array %s due to Metro Session Fracture", id, kubeNodeID, remoteArray.Endpoint) + } else { + log.Errorf("Failed to unpublish volume %s for array %s: %s", id, remoteArray.Endpoint, err) + return nil, unpublishErr } } else { - return nil, status.Errorf(codes.Internal, - "failure checking host '%s' status for volume unpublishing: %s", kubeNodeID, err.Error()) + log.Infof("Unpublished volume %s for array %s", id, remoteArray.Endpoint) + remoteVolumeUnpublished = true + response = resp } } + } + if !localVolumeUnpublished && !remoteVolumeUnpublished { + return nil, status.Error(codes.Internal, "failed to unpublish volume") + } - err = detachVolumeFromHost(ctx, node.ID, id, arr.GetClient()) - if err != nil { - return nil, err + if volumeHandle.IsMetro() && nodeConnectedToLocalArray && nodeConnectedToRemoteArray { + if (localVolumeUnpublished && !remoteVolumeUnpublished) || (!localVolumeUnpublished && remoteVolumeUnpublished) { + deferredRequest, err := proto.Marshal(req) + if err != nil { + log.Errorf("[METRO] Error marshalling req: %s", err.Error()) + return nil, err + } + + deferredArrayID := arrayID + if !remoteVolumeUnpublished { + deferredArrayID = remoteArrayID + } + + err = createOrUpdateJournalEntryFunc(ctx, metroResp.VolumeName, volumeHandle, deferredArrayID, kubeNodeID, "ControllerUnpublishVolume", deferredRequest) + if err != nil { + log.Errorf("Could not create journal entry for operation %s for volume %s node %s array %s", "ControllerUnpublishVolume", id, kubeNodeID, arrayID) + return nil, err + } + + log.Infof("[METRO] Metro volume %s created journal entry for operation %s for volume %s node %s array %s", id, "ControllerUnpublishVolume", id, kubeNodeID, arrayID) + } + } + + return response, nil +} + +func isNodeConnectedToArray(ctx context.Context, kubeNodeID string, arr *array.PowerStoreArray) bool { + return arr.CheckConnectivity(ctx, kubeNodeID) +} + +// unpublishVolume removes the mount to the target path and unpublishes the volume +func unpublishVolume(ctx context.Context, kubeNodeID string, arr *array.PowerStoreArray, volumeHandle *array.VolumeHandle, remoteArray *array.PowerStoreArray) (*csi.ControllerUnpublishVolumeResponse, error) { + if arr == nil && remoteArray == nil { + return &csi.ControllerUnpublishVolumeResponse{}, errors.New("no array information for controller unpublish") + } + + id := volumeHandle.LocalUUID + protocol := volumeHandle.Protocol + remoteVolumeID := volumeHandle.RemoteUUID + + switch protocol { + case "scsi": + // unpublish can be for remote array only + if arr != nil { + _, err := arr.GetClient().GetVolume(ctx, id) + if err != nil { + if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { + return &csi.ControllerUnpublishVolumeResponse{}, nil + } + } + node, err := arr.GetClient().GetHostByName(ctx, kubeNodeID) + if err != nil { + if apiError, ok := err.(gopowerstore.APIError); ok && apiError.HostIsNotExist() { + // We need additional check here since we can just have host without ip in it + ipList := identifiers.GetIPListFromString(kubeNodeID) + if ipList == nil { + return nil, errors.New("can't find IP in nodeID") + } + ip := ipList[len(ipList)-1] + nodeID := kubeNodeID[:len(kubeNodeID)-len(ip)-1] + node, err = arr.GetClient().GetHostByName(ctx, nodeID) + if err != nil { + return nil, status.Errorf(codes.NotFound, "host with k8s node ID '%s' not found", kubeNodeID) + } + } else { + return nil, status.Errorf(codes.Internal, + "failure checking host '%s' status for volume unpublishing: %s", kubeNodeID, err.Error()) + } + } + + err = detachVolumeFromHost(ctx, node.ID, id, arr.GetClient()) + if err != nil { + return nil, err + } + log.Debugf("Volume is unpublished on node %s for array %s", kubeNodeID, arr.Endpoint) } - if remoteArrayID != "" && remoteVolumeID != "" { // For Remote Metro volume + if remoteVolumeID != "" && remoteArray != nil { // For Remote Metro volume + _, err := remoteArray.GetClient().GetVolume(ctx, remoteVolumeID) + if err != nil { + if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { + return &csi.ControllerUnpublishVolumeResponse{}, nil + } + } node, err := remoteArray.GetClient().GetHostByName(ctx, kubeNodeID) if err != nil { return nil, status.Errorf(codes.Internal, @@ -866,10 +1213,11 @@ func (s *Service) ControllerUnpublishVolume(ctx context.Context, req *csi.Contro if err != nil { return nil, err } + log.Debugf("Volume is unpublished on node %s for array %s", kubeNodeID, remoteArray.Endpoint) } return &csi.ControllerUnpublishVolumeResponse{}, nil - } else if protocol == "nfs" { + case "nfs": fs, err := arr.GetClient().GetFS(ctx, id) if err != nil { if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { @@ -901,7 +1249,7 @@ func (s *Service) ControllerUnpublishVolume(ctx context.Context, req *csi.Contro if len(export.ROHosts) > 0 { if index >= 0 { modifyHostPayload.RemoveROHosts = []string{ip + "/255.255.255.255"} // we can't remove without netmask - log.Debug("Going to remove IP from ROHosts: ", modifyHostPayload.RemoveROHosts[0]) + log.Debugf("Going to remove IP from ROHosts: %s ", modifyHostPayload.RemoveROHosts[0]) } } @@ -910,24 +1258,24 @@ func (s *Service) ControllerUnpublishVolume(ctx context.Context, req *csi.Contro if len(export.RORootHosts) > 0 { if index >= 0 { modifyHostPayload.RemoveRORootHosts = []string{ip + "/255.255.255.255"} // we can't remove without netmask - log.Debug("Going to remove IP from RORootHosts: ", modifyHostPayload.RemoveRORootHosts[0]) + log.Debugf("Going to remove IP from RORootHosts: %s", modifyHostPayload.RemoveRORootHosts[0]) } } if identifiers.Contains(export.RWHosts, ip+"/255.255.255.255") { modifyHostPayload.RemoveRWHosts = []string{ip + "/255.255.255.255"} // we can't remove without netmask - log.Debug("Going to remove IP from RWHosts: ", modifyHostPayload.RemoveRWHosts[0]) + log.Debugf("Going to remove IP from RWHosts: %s", modifyHostPayload.RemoveRWHosts[0]) } if identifiers.Contains(export.RWRootHosts, ip+"/255.255.255.255") { modifyHostPayload.RemoveRWRootHosts = []string{ip + "/255.255.255.255"} // we can't remove without netmask - log.Debug("Going to remove IP from RWRootHosts: ", modifyHostPayload.RemoveRWRootHosts[0]) + log.Debugf("Going to remove IP from RWRootHosts: %s ", modifyHostPayload.RemoveRWRootHosts[0]) } // Detach host from nfs export _, err = arr.GetClient().ModifyNFSExport(ctx, &modifyHostPayload, export.ID) if err != nil { if apiError, ok := err.(gopowerstore.APIError); !(ok && apiError.HostAlreadyRemovedFromNFSExport()) { - log.Debug("Error occured while modifying NFS export during UnPublishVolume", err.Error()) + log.Debugf("Error occured while modifying NFS export during UnPublishVolume %s", err.Error()) return nil, status.Errorf(codes.Internal, "failure when removing new host to nfs export: %s", err.Error()) } @@ -947,38 +1295,39 @@ func GetServiceTag(ctx context.Context, req *csi.CreateVolumeRequest, arr *array var applianceName string var err error + log := log.WithContext(ctx) // Check if appliance id is present in PVC manifest if applianceID, ok := (req.Parameters)["appliance_id"]; ok { // Fetching appliance information using the appliance id ap, err = arr.Client.GetAppliance(ctx, applianceID) if err != nil { - log.Warn("Received error while calling GetAppliance ", err.Error()) + log.Warnf("Received error while calling GetAppliance %s", err.Error()) } } else { if protocol != "nfs" { vol, err = arr.Client.GetVolume(ctx, volID) if err != nil { - log.Warn("Received error while calling GetVolume ", err.Error()) + log.Warnf("Received error while calling GetVolume %s", err.Error()) } if vol.ApplianceID == "" { log.Warn("Unable to fetch ApplianceID from the volume") } else { ap, err = arr.Client.GetAppliance(ctx, vol.ApplianceID) if err != nil { - log.Warn("Received error while calling GetAppliance ", err.Error()) + log.Warnf("Received error while calling GetAppliance %s", err.Error()) } } } else { f, err = arr.Client.GetFS(ctx, volID) if err != nil { - log.Warn("Received error while calling GetFS ", err.Error()) + log.Warnf("Received error while calling GetFS %s", err.Error()) } if f.NasServerID == "" { log.Warn("Unable to fetch the NasServerID from the file system") } else { nas, err = arr.Client.GetNAS(ctx, f.NasServerID) if err != nil { - log.Warn("Received error while calling GetNAS ", err.Error()) + log.Warnf("Received error while calling GetNAS %s", err.Error()) } if nas.CurrentNodeID == "" { log.Warn("Unable to fetch the CurrentNodeId from the nas server") @@ -988,7 +1337,7 @@ func GetServiceTag(ctx context.Context, req *csi.CreateVolumeRequest, arr *array // Fetching appliance information using the appliance name ap, err = arr.Client.GetApplianceByName(ctx, applianceName) if err != nil { - log.Warn("Received error while calling GetApplianceByName ", err.Error()) + log.Warnf("Received error while calling GetApplianceByName %s", err.Error()) } } } @@ -1155,6 +1504,7 @@ func (s *Service) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) } func getMaximumVolumeSize(ctx context.Context, arr *array.PowerStoreArray) int64 { + log := log.WithContext(ctx) valueInCache, found := getCachedMaximumVolumeSize(arr.GlobalID) if !found || valueInCache < 0 { defaultHeaders := arr.Client.GetCustomHTTPHeaders() @@ -1615,13 +1965,15 @@ func (s *Service) RegisterAdditionalServers(server *grpc.Server) { } // ProbeController probes the controller service -func (s *Service) ProbeController(_ context.Context, _ *commonext.ProbeControllerRequest) (*commonext.ProbeControllerResponse, error) { +func (s *Service) ProbeController(ctx context.Context, _ *commonext.ProbeControllerRequest) (*commonext.ProbeControllerResponse, error) { + log := log.WithContext(ctx) ready := new(wrapperspb.BoolValue) ready.Value = true rep := new(commonext.ProbeControllerResponse) rep.Ready = ready rep.Name = identifiers.Name - rep.VendorVersion = core.SemVer + rep.VendorVersion = identifiers.ManifestSemver + identifiers.Manifest["semver"] = identifiers.ManifestSemver rep.Manifest = identifiers.Manifest log.Debug(fmt.Sprintf("ProbeController returning: %v", rep.Ready.GetValue())) @@ -1631,62 +1983,108 @@ func (s *Service) ProbeController(_ context.Context, _ *commonext.ProbeControlle func (s *Service) listPowerStoreVolumes(ctx context.Context, startToken, maxEntries int) ([]*csi.ListVolumesResponse_Entry, string, error) { var volResponse []*csi.ListVolumesResponse_Entry - // Get the volumes from the cache if we can - for _, arr := range s.Arrays() { - v, err := arr.GetClient().GetVolumes(ctx) + // Pre-fetch host-volume mappings for every array (so we only call mapping API once) + mappingsByArray := make(map[string][]gopowerstore.HostVolumeMapping) + for arrayID, arr := range s.Arrays() { + maps, err := arr.GetClient().GetHostVolumeMappings(ctx) + if err != nil { + log.Warnf("ListVolumes: failed to fetch host-volume mappings for array %s: %v", arrayID, err) + continue + } + mappingsByArray[arrayID] = maps + } + + // --------------------------- + // Block Volumes (SCSI) + // --------------------------- + for arrayID, arr := range s.Arrays() { + vols, err := arr.GetClient().GetVolumes(ctx) if err != nil { return nil, "", status.Errorf(codes.Internal, "unable to list volumes: %s", err.Error()) } - // Process the source volumes and make CSI Volumes - for _, vol := range v { - volResponse = append(volResponse, &csi.ListVolumesResponse_Entry{ - Volume: getCSIVolume(vol.ID, vol.Size), - }) + + // for each volume build CSI-style volume id and populate published node ids + maps := mappingsByArray[arrayID] + for _, vol := range vols { + // build CSI volumeID so it matches PV.spec.csi.volumeHandle + // format: "//scsi" + fullVolumeID := fmt.Sprintf("%s/%s/scsi", vol.ID, arrayID) + + entry := &csi.ListVolumesResponse_Entry{ + Volume: &csi.Volume{ + CapacityBytes: int64(vol.Size), + VolumeId: fullVolumeID, + }, + } + + // Populate PublishedNodeIds from host_volume_mapping -> host -> host.Name + var nodes []string + for _, m := range maps { + if m.VolumeID == vol.ID && m.HostID != "" { + if host, err := arr.GetClient().GetHost(ctx, m.HostID); err == nil && host.Name != "" { + nodes = append(nodes, host.Name) + } + } + } + if len(nodes) > 0 { + entry.Status = &csi.ListVolumesResponse_VolumeStatus{ + PublishedNodeIds: nodes, + } + } + volResponse = append(volResponse, entry) } } - // Get the FileSystems from the cache if we can - for _, arr := range s.Arrays() { - fs, err := arr.GetClient().ListFS(ctx) + // --------------------------- + // FileSystems (NFS) + // --------------------------- + for arrayID, arr := range s.Arrays() { + fsList, err := arr.GetClient().ListFS(ctx) if err != nil { - return nil, "", status.Errorf(codes.Internal, "unable to list Filesystems: %s", err.Error()) - } - // Process the source FileSystems and make CSI Volumes - for _, f := range fs { - volResponse = append(volResponse, &csi.ListVolumesResponse_Entry{ - Volume: getCSIVolume(f.ID, f.SizeTotal), - }) + return nil, "", status.Errorf(codes.Internal, "unable to list filesystems: %s", err.Error()) + } + for _, fs := range fsList { + // NFS volumeID format: "//nfs" + fullVolumeID := fmt.Sprintf("%s/%s/nfs", fs.ID, arrayID) + entry := &csi.ListVolumesResponse_Entry{ + Volume: &csi.Volume{ + CapacityBytes: int64(fs.SizeTotal), + VolumeId: fullVolumeID, + }, + } + // NFS does not have host mappings in the same way: leaving Status nil is fine + volResponse = append(volResponse, entry) } } + if startToken > len(volResponse) { return nil, "", status.Errorf(codes.Aborted, "startingToken=%d > len(volumes)=%d", startToken, len(volResponse)) } - // Discern the number of remaining entries. - rem := len(volResponse) - startToken + remaining := len(volResponse) - startToken - // If maxEntries is 0 or greater than the number of remaining entries then - // set max entries to the number of remaining entries. - if maxEntries == 0 || maxEntries > rem { - maxEntries = rem + if maxEntries == 0 || maxEntries > remaining { + maxEntries = remaining } - // We can't really return more per page + // Cap the number of entries returned in a single response. + // This prevents extremely large CSI responses, which can increase memory usage + // and impact ListVolumes performance on clusters with many volumes. if maxEntries > 700 { maxEntries = 700 } - // Compute the next starting point; if at end reset nextToken := startToken + maxEntries nextTokenStr := "" - if nextToken < (startToken + rem) { + if nextToken < len(volResponse) { nextTokenStr = fmt.Sprintf("%d", nextToken) } - return volResponse[startToken : startToken+maxEntries], nextTokenStr, nil + return volResponse[startToken:nextToken], nextTokenStr, nil } func (s *Service) listPowerStoreSnapshots(ctx context.Context, startToken, maxEntries int, snapID, srcID string) ([]GeneralSnapshot, string, error) { + log := log.WithContext(ctx) var generalSnapshots []GeneralSnapshot if snapID == "" && srcID == "" { @@ -1716,7 +2114,7 @@ func (s *Service) listPowerStoreSnapshots(ctx context.Context, startToken, maxEn log.Infof("Requested snapshot via snapshot id %s", snapID) volumeHandle, err := array.ParseVolumeID(ctx, snapID, s.DefaultArray(), nil) if err != nil { - log.Error(err) + log.Error(err.Error()) return []GeneralSnapshot{}, "", nil } @@ -1739,7 +2137,7 @@ func (s *Service) listPowerStoreSnapshots(ctx context.Context, startToken, maxEn return nil, "", status.Errorf(codes.Internal, "unable to get filesystem snapshot: %s", getErr.Error()) } - log.Info(fsSnapshot) + log.Infof("%+v", fsSnapshot) fsSnapshot.ID = fsSnapshot.ID + "/" + arrayID + "/" + protocol generalSnapshots = append(generalSnapshots, FilesystemSnapshot(fsSnapshot)) @@ -1760,7 +2158,7 @@ func (s *Service) listPowerStoreSnapshots(ctx context.Context, startToken, maxEn // This works VGS on single default array, But for multiple array scenario this default array should be changed to dynamic array volumeHandle, err := array.ParseVolumeID(ctx, srcID, s.DefaultArray(), nil) if err != nil { - log.Error(err) + log.Error(err.Error()) return []GeneralSnapshot{}, "", nil } diff --git a/pkg/controller/controller_node_to_array_connectivity.go b/pkg/controller/controller_node_to_array_connectivity.go index 31e94ace..8ed63399 100644 --- a/pkg/controller/controller_node_to_array_connectivity.go +++ b/pkg/controller/controller_node_to_array_connectivity.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2022-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2022-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,16 +27,15 @@ import ( "net/http" "time" - log "github.com/sirupsen/logrus" - "github.com/dell/csi-powerstore/v2/pkg/identifiers" ) // QueryArrayStatus make API call to the specified url to retrieve connection status func (s *Service) QueryArrayStatus(ctx context.Context, url string) (bool, error) { + log := log.WithContext(ctx) defer func() { if err := recover(); err != nil { - log.Println("panic occurred in queryStatus:", err) + log.Debugf("panic occurred in queryStatus: %v", err) } }() client := http.Client{ diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index 57b21cb5..2d430ff9 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -29,20 +29,29 @@ import ( "time" csiext "github.com/dell/dell-csi-extensions/replication" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/dell/csi-powerstore/v2/mocks" csictx "github.com/dell/gocsi/context" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/pkg/array" "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/k8sutils" "github.com/dell/gopowerstore" "github.com/dell/gopowerstore/api" gopowerstoremock "github.com/dell/gopowerstore/mocks" + "github.com/container-storage-interface/spec/lib/go/csi" ginkgo "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" gomega "github.com/onsi/gomega" "github.com/stretchr/testify/mock" + k8score "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" ) const ( @@ -81,6 +90,7 @@ const ( validApplianceID = "my-appliance" validRemoteApplianceID = "my-appliance2" validServiceTag = "service-tag" + replicationSessionID = "123456" ) var ( @@ -133,6 +143,21 @@ var ( ) func TestCSIControllerService(t *testing.T) { + defaultK8sConfigFunc := k8sutils.InClusterConfigFunc + defaultK8sClientsetFunc := k8sutils.NewForConfigFunc + + k8sutils.InClusterConfigFunc = func() (*rest.Config, error) { + return &rest.Config{}, nil + } + k8sutils.NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return fake.NewClientset(), nil + } + + defer func() { + k8sutils.InClusterConfigFunc = defaultK8sConfigFunc + k8sutils.NewForConfigFunc = defaultK8sClientsetFunc + }() + gomega.RegisterFailHandler(ginkgo.Fail) junitReporter := reporters.NewJUnitReporter("ctrl-svc.xml") ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "CSIControllerService testing suite", []ginkgo.Reporter{junitReporter}) @@ -174,9 +199,15 @@ func setVariables() { csictx.Setenv(context.Background(), identifiers.EnvReplicationPrefix, "replication.storage.dell.com") csictx.Setenv(context.Background(), identifiers.EnvNfsAcls, "A::OWNER@:RWX") - ctrlSvc = &Service{Fs: fsMock} + ctrlSvc = &Service{ + Fs: fsMock, + IsCSMDREnabled: true, + } ctrlSvc.SetArrays(arrays) ctrlSvc.SetDefaultArray(first) + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewSimpleClientset(), + } ctrlSvc.Init() } @@ -982,6 +1013,27 @@ var _ = ginkgo.Describe("CSIControllerService", func() { })) }) + ginkgo.It("should configure not metro replication on volume due to CSM DR not avilable", func() { + // setting the value of IsCSMDREnabled parameter of ControllerService to false + ctrlSvc.IsCSMDREnabled = false + _ = ctrlSvc.Init() + delete(req.Parameters, ctrlSvc.WithRP(KeyReplicationRPO)) + delete(req.Parameters, ctrlSvc.WithRP(KeyReplicationIgnoreNamespaces)) + delete(req.Parameters, ctrlSvc.WithRP(KeyReplicationVGPrefix)) + + clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("no vol found")) + ctrlSvc.IsCSMDREnabled = false + res, err := ctrlSvc.CreateVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("Metro replication mode requires CSM-DR to be enabled")) + // Resetting the value of IsCSMDREnabled parameter of ControllerService to true after our tests are completed + ctrlSvc.IsCSMDREnabled = true + _ = ctrlSvc.Init() + }) + ginkgo.It("should fail to configure metro replication on volume if the volume cannot be found", func() { clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( gopowerstore.Volume{}, errors.New("no vol found")) @@ -1847,20 +1899,6 @@ var _ = ginkgo.Describe("CSIControllerService", func() { )) }) }) - - ginkgo.When("nfs replication", func() { - ginkgo.It("should fail", func() { - req := getTypicalCreateVolumeNFSRequest("my-vol", validVolSize) - req.Parameters[identifiers.KeyArrayID] = secondValidID - req.Parameters[ctrlSvc.WithRP(KeyReplicationEnabled)] = "true" - - res, err := ctrlSvc.CreateVolume(context.Background(), req) - - gomega.Expect(res).To(gomega.BeNil()) - gomega.Expect(err).NotTo(gomega.BeNil()) - gomega.Expect(err.Error()).To(gomega.ContainSubstring("replication not supported for NFS")) - }) - }) }) ginkgo.When("multi-nas is enabled for NFS", func() { @@ -2392,6 +2430,12 @@ var _ = ginkgo.Describe("CSIControllerService", func() { clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{}, gopowerstore.APIError{ ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusGatewayTimeout}, }) + clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return( + gopowerstore.Volume{ + ID: validBaseVolID, + Name: "name", + Size: validVolSize, + }, nil) req := &csi.DeleteVolumeRequest{VolumeId: validBlockVolumeID} res, err := ctrlSvc.DeleteVolume(context.Background(), req) @@ -2565,6 +2609,205 @@ var _ = ginkgo.Describe("CSIControllerService", func() { gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't figure out protocol")) }) }) + + ginkgo.When("MetroFractured: deleting metro volume when remote array is down", func() { + ginkgo.It("should succeed fail", func() { + // local info + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, mock.Anything).Return([]gopowerstore.Volume{}, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, mock.Anything).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("DeleteVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeDelete"), + validBaseVolID). + Return(gopowerstore.EmptyResponse(""), nil) + clientMock.On("EndMetroVolume", mock.Anything, mock.Anything, mock.Anything).Return(gopowerstore.EmptyResponse(""), nil) + + // remote info + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", ApplianceID: validRemoteApplianceID}, errors.New("timeout")) + clientMock.On("DeleteVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeDelete"), + validRemoteVolID). + Return(gopowerstore.EmptyResponse(""), errors.New("timeout")) + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.DeleteVolumeRequest{VolumeId: volumeID} + + res, err := ctrlSvc.DeleteVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(res).To(gomega.BeNil()) + }) + }) + + ginkgo.When("MetroFractured: when local array is down", func() { + ginkgo.It("DeleteVolume should fail ", func() { + // local info + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, mock.Anything).Return([]gopowerstore.Volume{}, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, mock.Anything).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("DeleteVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeDelete"), + validBaseVolID). + Return(gopowerstore.EmptyResponse(""), errors.New("timeout")) + clientMock.On("EndMetroVolume", mock.Anything, mock.Anything, mock.Anything).Return(gopowerstore.EmptyResponse(""), nil) + + // remote info + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", ApplianceID: validRemoteApplianceID}, errors.New("timeout")) + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.DeleteVolumeRequest{VolumeId: volumeID} + + res, err := ctrlSvc.DeleteVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(res).To(gomega.BeNil()) + }) + }) + + ginkgo.When("MetroFractured: no array is down", func() { + ginkgo.It("should succeed", func() { + // local info + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", ApplianceID: validApplianceID, MetroReplicationSessionID: replicationSessionID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{}, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("DeleteVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeDelete"), + validBaseVolID). + Return(gopowerstore.EmptyResponse(""), nil) + clientMock.On("EndMetroVolume", mock.Anything, mock.Anything, mock.Anything).Return(gopowerstore.EmptyResponse(""), nil) + + // remote info + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", ApplianceID: validRemoteApplianceID, MetroReplicationSessionID: replicationSessionID}, nil) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validRemoteVolID).Return([]gopowerstore.Volume{}, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validRemoteVolID).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("DeleteVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeDelete"), + validRemoteVolID). + Return(gopowerstore.EmptyResponse(""), nil) + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.DeleteVolumeRequest{VolumeId: volumeID} + + res, err := ctrlSvc.DeleteVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.DeleteVolumeResponse{})) + }) + + ginkgo.It("succeed: metro only contains promote/demote snapshots", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", ApplianceID: validApplianceID, MetroReplicationSessionID: replicationSessionID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Ok", + LocalResourceState: "Promoted", + }, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{{ID: "snap-id-1", Name: "Metro_Promote_Backup_Snapshot." + "vol1"}}, nil) + clientMock.On("EndMetroVolume", mock.Anything, mock.Anything, mock.Anything).Return(gopowerstore.EmptyResponse(""), nil) + clientMock.On("DeleteVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeDelete"), + validBaseVolID). + Return(gopowerstore.EmptyResponse(""), nil) + req := &csi.DeleteVolumeRequest{VolumeId: validBaseVolID} + res, err := ctrlSvc.DeleteVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.DeleteVolumeResponse{})) + }) + + ginkgo.It("failure: metro volume contains snapshot not promote/demote", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", ApplianceID: validApplianceID, MetroReplicationSessionID: replicationSessionID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Ok", + LocalResourceState: "Promoted", + }, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{{ID: "snap-id-1", Name: "myLocalSnapshot"}}, nil) + clientMock.On("EndMetroVolume", mock.Anything, mock.Anything, mock.Anything).Return(gopowerstore.EmptyResponse(""), nil) + req := &csi.DeleteVolumeRequest{VolumeId: validBaseVolID} + res, err := ctrlSvc.DeleteVolume(context.Background(), req) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("snapshots based on this volume still exist")) + }) + }) + + ginkgo.When("MetroFractured: both array down", func() { + ginkgo.It("should fail", func() { + // local info + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", ApplianceID: validApplianceID, MetroReplicationSessionID: replicationSessionID}, errors.New("timeout")) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{}, errors.New("timeout")) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, errors.New("timeout")) + clientMock.On("DeleteVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeDelete"), + validBaseVolID). + Return(gopowerstore.EmptyResponse(""), errors.New("timeout")) + clientMock.On("EndMetroVolume", mock.Anything, mock.Anything, mock.Anything).Return(gopowerstore.EmptyResponse(""), errors.New("timeout")) + + // remote info + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", ApplianceID: validRemoteApplianceID, MetroReplicationSessionID: replicationSessionID}, errors.New("timeout")) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validRemoteVolID).Return([]gopowerstore.Volume{}, errors.New("timeout")) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validRemoteVolID).Return(gopowerstore.VolumeGroups{}, errors.New("timeout")) + clientMock.On("DeleteVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeDelete"), + validRemoteVolID). + Return(gopowerstore.EmptyResponse(""), errors.New("timeout")) + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.DeleteVolumeRequest{VolumeId: volumeID} + + res, err := ctrlSvc.DeleteVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(res).To(gomega.BeNil()) + }) + }) }) ginkgo.Describe("calling CreateSnapshot()", func() { @@ -3326,6 +3569,78 @@ var _ = ginkgo.Describe("CSIControllerService", func() { gomega.Expect(err).To(gomega.BeNil()) _ = ctrlSvc.Init() }) + + ginkgo.It("should succeed [NFS] with externalAccess and exclusiveAccess enabled", func() { + // setting externalAccess and exclusiveAccess environment variable + err := csictx.Setenv(context.Background(), identifiers.EnvExternalAccess, "10.0.0.0/24") + gomega.Expect(err).To(gomega.BeNil()) + err = csictx.Setenv(context.Background(), identifiers.EnvExclusiveAccess, "true") + gomega.Expect(err).To(gomega.BeNil()) + _ = ctrlSvc.Init() + + clientMock.On("GetFS", mock.Anything, validBaseVolID). + Return(gopowerstore.FileSystem{ + ID: validBaseVolID, + Name: fsName, + NasServerID: nasID, + }, nil) + + apiError := gopowerstore.NewAPIError() + apiError.StatusCode = http.StatusNotFound + + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, mock.Anything). + Return(gopowerstore.NFSExport{}, *apiError).Once() + + nfsExportCreate := &gopowerstore.NFSExportCreate{ + Name: fsName, + FileSystemID: validBaseVolID, + Path: "/" + fsName, + } + clientMock.On("CreateNFSExport", mock.Anything, nfsExportCreate). + Return(gopowerstore.CreateResponse{ID: nfsID}, nil) + + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, mock.Anything). + Return(gopowerstore.NFSExport{ID: nfsID}, nil).Once() + + clientMock.On("ModifyNFSExport", mock.Anything, &gopowerstore.NFSExportModify{ + AddRWRootHosts: []string{ + "10.0.0.0/255.255.255.0", + }, + }, nfsID).Return(gopowerstore.CreateResponse{}, nil) + + clientMock.On("GetNAS", mock.Anything, nasID). + Return(gopowerstore.NAS{ + Name: validNasName, + CurrentPreferredIPv4InterfaceID: interfaceID, + }, nil) + + clientMock.On("GetFileInterface", mock.Anything, interfaceID). + Return(gopowerstore.FileInterface{IPAddress: secondValidID}, nil) + + req := getTypicalControllerPublishVolumeRequest("multiple-writer", validNodeID, validNfsVolumeID) + req.VolumeCapability = getVolumeCapabilityNFS() + req.VolumeContext = map[string]string{KeyFsType: "nfs"} + + res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ + PublishContext: map[string]string{ + identifiers.KeyNasName: validNasName, + identifiers.KeyNfsExportPath: secondValidID + ":/", + identifiers.KeyExportID: nfsID, + identifiers.KeyAllowRoot: "", + identifiers.KeyNfsACL: "", + identifiers.KeyNatIP: "10.0.0.0/255.255.255.0", + }, + })) + // Removing externalAccess and exclusiveAccess environment variable after our tests are completed + err = csictx.Setenv(context.Background(), identifiers.EnvExternalAccess, "") + gomega.Expect(err).To(gomega.BeNil()) + err = csictx.Setenv(context.Background(), identifiers.EnvExclusiveAccess, "") + gomega.Expect(err).To(gomega.BeNil()) + _ = ctrlSvc.Init() + }) }) ginkgo.When("host name does not contain ip", func() { @@ -3719,19 +4034,668 @@ var _ = ginkgo.Describe("CSIControllerService", func() { }) }) - ginkgo.When("volume id is empty", func() { + ginkgo.When("When attach fails for a non-metro volume ", func() { ginkgo.It("should fail", func() { - req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) + // local info + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", ApplianceID: validApplianceID}, nil) + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.HostVolumeMapping{}, nil).Once() + clientMock.On("AttachVolumeToHost", mock.Anything, validHostID, mock.Anything). + Return(gopowerstore.EmptyResponse(""), errors.New("some error")).Times(2) - req.VolumeId = "" + volumeID := fmt.Sprintf("%s/%s/%s", validBaseVolID, firstValidID, "scsi") + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + req.VolumeContext = map[string]string{KeyFsType: "xfs"} - _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) - gomega.Expect(err).ToNot(gomega.BeNil()) - gomega.Expect(err.Error()).To(gomega.ContainSubstring("volume ID is required")) + res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(res).To(gomega.BeNil()) }) }) - ginkgo.When("volume capability is missing", func() { + ginkgo.When("MetroFractured: publishing metro volume when one array is down", func() { + ginkgo.It("should succeed [Block] when remote array is down and local was promoted", func() { + // local info + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.HostVolumeMapping{}, nil).Once() + clientMock.On("AttachVolumeToHost", mock.Anything, validHostID, mock.Anything). + Return(gopowerstore.EmptyResponse(""), nil).Times(2) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID, LogicalUnitNumber: 1}}, nil).Once() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + ApplianceID: validApplianceID, + }, + }, nil).Once() + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + ApplianceID: validApplianceID, + }, + }, nil).Times(2) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName, NVMeNQN: "nqn"}, nil).Times(2) + clientMock.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{ + { + IsLinkUp: true, + Wwn: "58:cc:f0:93:48:a0:03:a3", + WwnNVMe: "58ccf091492b0c22", + WwnNode: "58ccf090c9200c22", + ApplianceID: validApplianceID, + }, + }, nil).Times(2) + + // remote info + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", ApplianceID: validRemoteApplianceID}, errors.New("timeout")) + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + req.VolumeContext = map[string]string{KeyFsType: "xfs"} + + res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ + PublishContext: map[string]string{ + "DEVICE_WWN": "68ccf098003ceb5e4577a20be6d11bf9", + "LUN_ADDRESS": "1", + }, + })) + }) + }) + + ginkgo.When("MetroFractured: when local volume was promoted, but local publish failed", func() { + ginkgo.It("publishVolume should fail ", func() { + // local info + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.HostVolumeMapping{}, nil).Once() + clientMock.On("AttachVolumeToHost", mock.Anything, validHostID, mock.Anything). + Return(gopowerstore.EmptyResponse(""), errors.New("some error")).Times(2) + + // remote info + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(nil, errors.New("timeout")) + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + req.VolumeContext = map[string]string{KeyFsType: "xfs"} + + res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(res).To(gomega.BeNil()) + }) + }) + + ginkgo.When("MetroFractured: remote volume was promoted and local array was down", func() { + ginkgo.It("should succeed", func() { + // local info + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, errors.New("timeout")) + + // remote info + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validRemoteApplianceID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validRemoteVolID). + Return([]gopowerstore.HostVolumeMapping{}, nil).Once() + clientMock.On("AttachVolumeToHost", mock.Anything, validHostID, mock.Anything). + Return(gopowerstore.EmptyResponse(""), nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validRemoteVolID). + Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID, LogicalUnitNumber: 1}}, nil).Once() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn.2015-10.com.dell:dellemc-powerstore-apm00223"}, + ApplianceID: validRemoteApplianceID, + }, + }, nil).Once() + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn.2015-10.com.dell:dellemc-powerstore-apm00223"}, + ApplianceID: validRemoteApplianceID, + }, + }, nil).Times(2) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName, NVMeNQN: "nqn.1988-11.com.dell:powerstore:00:303030303030ABCDEFGH"}, nil).Times(2) + clientMock.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{ + { + IsLinkUp: true, + Wwn: "58:cc:f0:93:48:a0:03:33", + WwnNVMe: "58ccf091492b0c33", + WwnNode: "58ccf090c9200c33", + ApplianceID: validRemoteApplianceID, + }, + }, nil).Times(2) + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + req.VolumeContext = map[string]string{KeyFsType: "xfs"} + + res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ + PublishContext: map[string]string{ + "REMOTE_DEVICE_WWN": "68ccf098003ceb5e4577a20be6d11bf9", + "REMOTE_LUN_ADDRESS": "1", + }, + })) + }) + }) + + ginkgo.When("MetroFractured: when remote volume was promoted, but remote publish failed", func() { + ginkgo.It("publishVolume should fail ", func() { + // local info + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Demoted", + }, nil) + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.HostVolumeMapping{}, nil) + clientMock.On("AttachVolumeToHost", mock.Anything, validHostID, mock.Anything). + Return(gopowerstore.EmptyResponse(""), errors.New("some error")).Times(2) + + // remote info + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validRemoteApplianceID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validRemoteVolID). + Return([]gopowerstore.HostVolumeMapping{}, nil) + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + req.VolumeContext = map[string]string{KeyFsType: "xfs"} + + res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(res).To(gomega.BeNil()) + }) + }) + + ginkgo.When("publishing metro volume with locator", func() { + ginkgo.BeforeEach(func() { + // local info + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", ApplianceID: validApplianceID}, nil) + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.HostVolumeMapping{}, nil).Once() + clientMock.On("AttachVolumeToHost", mock.Anything, validHostID, mock.Anything). + Return(gopowerstore.EmptyResponse(""), nil).Times(2) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID, LogicalUnitNumber: 1}}, nil).Once() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + ApplianceID: validApplianceID, + }, + }, nil).Once() + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + ApplianceID: validApplianceID, + }, + }, nil).Times(2) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName, NVMeNQN: "nqn"}, nil).Times(2) + clientMock.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{ + { + IsLinkUp: true, + Wwn: "58:cc:f0:93:48:a0:03:a3", + WwnNVMe: "58ccf091492b0c22", + WwnNode: "58ccf090c9200c22", + ApplianceID: validApplianceID, + }, + }, nil).Times(2) + + // remote info + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", ApplianceID: validRemoteApplianceID}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validRemoteVolID). + Return([]gopowerstore.HostVolumeMapping{}, nil).Once() + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validRemoteVolID). + Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID, LogicalUnitNumber: 1}}, nil).Once() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn.2015-10.com.dell:dellemc-powerstore-apm00223"}, + ApplianceID: validRemoteApplianceID, + }, + }, nil).Once() + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn.2015-10.com.dell:dellemc-powerstore-apm00223"}, + ApplianceID: validRemoteApplianceID, + }, + }, nil).Times(2) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName, NVMeNQN: "nqn.1988-11.com.dell:powerstore:00:303030303030ABCDEFGH"}, nil).Times(2) + clientMock.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{ + { + IsLinkUp: true, + Wwn: "58:cc:f0:93:48:a0:03:33", + WwnNVMe: "58ccf091492b0c33", + WwnNode: "58ccf090c9200c33", + ApplianceID: validRemoteApplianceID, + }, + }, nil).Times(2) + + metroArr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"nonu1"}, + }, + }, + }, + }, + }, + + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone2"}, + }, + }, + }, + }, + }, + ColocatedBoth: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone3"}, + }, + }, + }, + }, + }, + }, + }, + GlobalID: "APM0012345", + Endpoint: "https://192.168.0.3/api/rest", + Username: "admin", + Password: "pass", + BlockProtocol: identifiers.ISCSITransport, + Insecure: true, + Client: clientMock, + IP: "192.168.0.3", + NASCooldownTracker: array.NewNASCooldown(time.Minute, 5), + } + metroRemoteArr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"nonu2"}, + }, + }, + }, + }, + }, + + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone2"}, + }, + }, + }, + }, + }, + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + ColocatedBoth: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone3"}, + }, + }, + }, + }, + }, + }, + }, + GlobalID: "APM0045678", + Endpoint: "https://192.168.1.3/api/rest", + Username: "admin", + Password: "pass", + BlockProtocol: identifiers.ISCSITransport, + Insecure: true, + Client: clientMock, + IP: "192.168.1.3", + NASCooldownTracker: array.NewNASCooldown(time.Minute, 5), + } + arrays := ctrlSvc.Arrays() + arrays["APM0012345"] = metroArr + arrays["APM0045678"] = metroRemoteArr + ctrlSvc.SetArrays(arrays) + }) + + ginkgo.It("should pass with node selector passing [Block]", func() { + // Create a node that matches the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "zone1", // matches the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{node}...), + } + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") + + // Publish the volume + publishReq := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + publishReq.VolumeContext = map[string]string{ + KeyFsType: "xfs", + ctrlSvc.WithRP(KeyReplicationMode): "METRO", + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationRPO): zeroRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + } + + publishResp, err := ctrlSvc.ControllerPublishVolume(context.Background(), publishReq) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(publishResp).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ + PublishContext: map[string]string{ + "DEVICE_WWN": "68ccf098003ceb5e4577a20be6d11bf9", + "LUN_ADDRESS": "1", + "REMOTE_DEVICE_WWN": "68ccf098003ceb5e4577a20be6d11bf9", + "REMOTE_LUN_ADDRESS": "1", + }, + })) + }) + + ginkgo.It("should pass with local node selector passing [Block]", func() { + // Create a node that matches the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "nonu1", // matches the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + k8sutils.Kubeclient.Clientset.CoreV1().Nodes().Create(context.Background(), node, metav1.CreateOptions{}) + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") + + // Publish the volume + publishReq := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + publishReq.VolumeContext = map[string]string{ + KeyFsType: "xfs", + ctrlSvc.WithRP(KeyReplicationMode): "METRO", + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationRPO): zeroRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + } + + publishResp, err := ctrlSvc.ControllerPublishVolume(context.Background(), publishReq) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(publishResp).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ + PublishContext: map[string]string{ + "DEVICE_WWN": "68ccf098003ceb5e4577a20be6d11bf9", + "LUN_ADDRESS": "1", + }, + })) + }) + ginkgo.It("should pass with local node selector of remote array passing [Block]", func() { + // Create a node that matches the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "nonu2", // matches the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{node}...), + } + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") + + // Publish the volume + publishReq := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + publishReq.VolumeContext = map[string]string{ + KeyFsType: "xfs", + ctrlSvc.WithRP(KeyReplicationMode): "METRO", + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationRPO): zeroRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + } + + publishResp, err := ctrlSvc.ControllerPublishVolume(context.Background(), publishReq) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(publishResp).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ + PublishContext: map[string]string{ + "REMOTE_DEVICE_WWN": "68ccf098003ceb5e4577a20be6d11bf9", + "REMOTE_LUN_ADDRESS": "1", + }, + })) + }) + + ginkgo.It("should fail due to node selector mismatch [Block]", func() { + // Create a node that does not match the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "zone5", // does not match the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{node}...), + } + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") + + // Publish the volume + publishReq := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + publishReq.VolumeContext = map[string]string{ + KeyFsType: "xfs", + ctrlSvc.WithRP(KeyReplicationMode): "METRO", + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationRPO): zeroRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + } + + _, err := ctrlSvc.ControllerPublishVolume(context.Background(), publishReq) + gomega.Expect(err).To(gomega.HaveOccurred()) + }) + ginkgo.It("should fail due to no node presence [Block]", func() { + k8sutils.Kubeclient = &k8sutils.K8sClient{} + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") + + // Publish the volume + publishReq := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + publishReq.VolumeContext = map[string]string{ + KeyFsType: "xfs", + ctrlSvc.WithRP(KeyReplicationMode): "METRO", + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationRPO): zeroRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + } + + _, err := ctrlSvc.ControllerPublishVolume(context.Background(), publishReq) + gomega.Expect(err).To(gomega.HaveOccurred()) + }) + ginkgo.It("should fail due to no remote system [Block]", func() { + // Create a node that matches the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "zone1", // matches the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{node}...), + } + clientMock.On("GetVolume", mock.Anything, validRemoteVolID+"1"). + Return(gopowerstore.Volume{}, fmt.Errorf("failed to get volume")) + // Invalid remote system + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID+"1", "APM0456789") + + // Publish the volume + publishReq := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + publishReq.VolumeContext = map[string]string{ + KeyFsType: "xfs", + ctrlSvc.WithRP(KeyReplicationMode): "METRO", + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationRPO): zeroRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + } + + _, err := ctrlSvc.ControllerPublishVolume(context.Background(), publishReq) + gomega.Expect(err).To(gomega.HaveOccurred()) + }) + }) + + ginkgo.When("volume id is empty", func() { + ginkgo.It("should fail", func() { + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) + + req.VolumeId = "" + + _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("volume ID is required")) + }) + }) + + ginkgo.When("volume capability is missing", func() { ginkgo.It("should fail", func() { req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) @@ -3884,6 +4848,9 @@ var _ = ginkgo.Describe("CSIControllerService", func() { }, }).Once() + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9"}, nil) + clientMock.On("GetHostByName", mock.Anything, validHostName). Return(gopowerstore.Host{ID: validHostID}, nil).Once() @@ -4034,23 +5001,534 @@ var _ = ginkgo.Describe("CSIControllerService", func() { Return(gopowerstore.Host{ID: validHostID}, nil).Times(2) clientMock.On("DetachVolumeFromHost", mock.Anything, mock.Anything, mock.Anything). Return(gopowerstore.EmptyResponse(""), nil).Times(2) + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Active-Active", + LocalResourceState: "System-Defined", + }, nil) + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, nil) + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) + }) + + ginkgo.It("should fail", func() { + ip := "127.0.0.1" // we don't have array with this IP + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, ip) + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("cannot find remote array")) + }) + + ginkgo.It("should succeed when volume not found on local or remote array", func() { + clientMock.On("GetHostByName", mock.Anything, mock.Anything).Return(gopowerstore.Host{ID: validHostID}, nil) + originalCheckMetroStateFunc := checkMetroStateFunc + checkMetroStateFunc = func(_ context.Context, _ array.VolumeHandle, _ gopowerstore.Client, _ gopowerstore.Client) (*array.MetroFracturedResponse, bool, error) { + return nil, false, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, + }, + } + } + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true // mocking uniform metro + } + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, + }, + }) + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, + }, + }) + defer func() { + isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc + checkMetroStateFunc = originalCheckMetroStateFunc + }() + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) + }) + + ginkgo.It("should fail due to checkMetroStateFunc returning error", func() { + clientMock.On("GetHostByName", mock.Anything, mock.Anything).Return(gopowerstore.Host{ID: validHostID}, nil) + originalCheckMetroStateFunc := checkMetroStateFunc + checkMetroStateFunc = func(_ context.Context, _ array.VolumeHandle, _ gopowerstore.Client, _ gopowerstore.Client) (*array.MetroFracturedResponse, bool, error) { + return nil, false, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusBadGateway, + }, + } + } + defer func() { + checkMetroStateFunc = originalCheckMetroStateFunc + }() + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + gomega.Expect(err).ToNot(gomega.BeNil()) + }) + + ginkgo.It("should succeed when metro is fractured, remote array unpublish fails and local array was promoted", func() { + clientMock.On("GetHostByName", mock.Anything, mock.Anything).Return(gopowerstore.Host{ID: validHostID}, nil) + originalCheckMetroStateFunc := checkMetroStateFunc + checkMetroStateFunc = func(_ context.Context, _ array.VolumeHandle, _ gopowerstore.Client, _ gopowerstore.Client) (*array.MetroFracturedResponse, bool, error) { + return &array.MetroFracturedResponse{IsFractured: true, State: "Promoted", VolumeName: "vol1"}, false, nil + } + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true // mocking uniform metro + } + originalUnpublishVolumeFunc := unpublishVolumeFunc + unpublishVolumeFunc = func(_ context.Context, _ string, arr *array.PowerStoreArray, _ *array.VolumeHandle, _ *array.PowerStoreArray) (*csi.ControllerUnpublishVolumeResponse, error) { + if arr != nil { + return &csi.ControllerUnpublishVolumeResponse{}, nil + } + return nil, errors.New("some-error") + } + originalCreateOrUpdateJournalEntryFunc := createOrUpdateJournalEntryFunc + createOrUpdateJournalEntryFunc = func(_ context.Context, _ string, _ array.VolumeHandle, _ string, _ string, _ string, _ []byte) error { + return nil + } + defer func() { + isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc + unpublishVolumeFunc = originalUnpublishVolumeFunc + checkMetroStateFunc = originalCheckMetroStateFunc + createOrUpdateJournalEntryFunc = originalCreateOrUpdateJournalEntryFunc + }() + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) + }) + + ginkgo.It("should fail when metro is fractured, remote array unpublish fails and local array was demoted", func() { + clientMock.On("GetHostByName", mock.Anything, mock.Anything).Return(gopowerstore.Host{ID: validHostID}, nil) + originalCheckMetroStateFunc := checkMetroStateFunc + checkMetroStateFunc = func(_ context.Context, _ array.VolumeHandle, _ gopowerstore.Client, _ gopowerstore.Client) (*array.MetroFracturedResponse, bool, error) { + return &array.MetroFracturedResponse{IsFractured: true, State: "Demoted", VolumeName: "vol1"}, true, nil + } + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true // mocking uniform metro + } + originalUnpublishVolumeFunc := unpublishVolumeFunc + unpublishVolumeFunc = func(_ context.Context, _ string, arr *array.PowerStoreArray, _ *array.VolumeHandle, _ *array.PowerStoreArray) (*csi.ControllerUnpublishVolumeResponse, error) { + if arr != nil { + return &csi.ControllerUnpublishVolumeResponse{}, nil + } + return nil, errors.New("some-error") + } + defer func() { + isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc + unpublishVolumeFunc = originalUnpublishVolumeFunc + checkMetroStateFunc = originalCheckMetroStateFunc + }() + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(res).To(gomega.BeNil()) + }) + + ginkgo.It("should succeed when metro is fractured, local array unpublish fails and remote array was promoted", func() { + clientMock.On("GetHostByName", mock.Anything, mock.Anything).Return(gopowerstore.Host{ID: validHostID}, nil) + originalCheckMetroStateFunc := checkMetroStateFunc + checkMetroStateFunc = func(_ context.Context, _ array.VolumeHandle, _ gopowerstore.Client, _ gopowerstore.Client) (*array.MetroFracturedResponse, bool, error) { + return &array.MetroFracturedResponse{IsFractured: true, State: "Demoted", VolumeName: "vol1"}, true, nil + } + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true // mocking uniform metro + } + originalUnpublishVolumeFunc := unpublishVolumeFunc + unpublishVolumeFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray, _ *array.VolumeHandle, remoteArray *array.PowerStoreArray) (*csi.ControllerUnpublishVolumeResponse, error) { + if remoteArray != nil { + return &csi.ControllerUnpublishVolumeResponse{}, nil + } + return nil, errors.New("some-error") + } + originalCreateOrUpdateJournalEntryFunc := createOrUpdateJournalEntryFunc + createOrUpdateJournalEntryFunc = func(_ context.Context, _ string, _ array.VolumeHandle, _ string, _ string, _ string, _ []byte) error { + return nil + } + defer func() { + isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc + unpublishVolumeFunc = originalUnpublishVolumeFunc + checkMetroStateFunc = originalCheckMetroStateFunc + createOrUpdateJournalEntryFunc = originalCreateOrUpdateJournalEntryFunc + }() + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) + }) + + ginkgo.It("should fail when metro is fractured, local array unpublish fails and local array was promoted", func() { + clientMock.On("GetHostByName", mock.Anything, mock.Anything).Return(gopowerstore.Host{ID: validHostID}, nil) + originalCheckMetroStateFunc := checkMetroStateFunc + checkMetroStateFunc = func(_ context.Context, _ array.VolumeHandle, _ gopowerstore.Client, _ gopowerstore.Client) (*array.MetroFracturedResponse, bool, error) { + return &array.MetroFracturedResponse{IsFractured: true, State: "Promoted", VolumeName: "vol1"}, false, nil + } + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true // mocking uniform metro + } + originalUnpublishVolumeFunc := unpublishVolumeFunc + unpublishVolumeFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray, _ *array.VolumeHandle, remoteArray *array.PowerStoreArray) (*csi.ControllerUnpublishVolumeResponse, error) { + if remoteArray != nil { + return &csi.ControllerUnpublishVolumeResponse{}, nil + } + return nil, errors.New("some-error") + } + originalCreateOrUpdateJournalEntryFunc := createOrUpdateJournalEntryFunc + createOrUpdateJournalEntryFunc = func(_ context.Context, _ string, _ array.VolumeHandle, _ string, _ string, _ string, _ []byte) error { + return nil + } + defer func() { + isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc + unpublishVolumeFunc = originalUnpublishVolumeFunc + checkMetroStateFunc = originalCheckMetroStateFunc + createOrUpdateJournalEntryFunc = originalCreateOrUpdateJournalEntryFunc + }() volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(res).To(gomega.BeNil()) + }) + }) + + ginkgo.When("unpublishing metro volume with selector", func() { + ginkgo.BeforeEach(func() { + // local info + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + clientMock.On("DetachVolumeFromHost", mock.Anything, mock.Anything, mock.Anything). + Return(gopowerstore.EmptyResponse(""), nil).Times(2) + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, nil) + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Active-Active", + LocalResourceState: "System-Defined", + }, nil) + + metroArr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"nonu1"}, + }, + }, + }, + }, + }, + + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone2"}, + }, + }, + }, + }, + }, + ColocatedBoth: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone3"}, + }, + }, + }, + }, + }, + }, + }, + GlobalID: "APM0012345", + Endpoint: "https://192.168.0.3/api/rest", + Username: "admin", + Password: "pass", + BlockProtocol: identifiers.ISCSITransport, + Insecure: true, + Client: clientMock, + IP: "192.168.0.3", + NASCooldownTracker: array.NewNASCooldown(time.Minute, 5), + } + metroRemoteArr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"nonu2"}, + }, + }, + }, + }, + }, + + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone2"}, + }, + }, + }, + }, + }, + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + ColocatedBoth: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone3"}, + }, + }, + }, + }, + }, + }, + }, + GlobalID: "APM0045678", + Endpoint: "https://192.168.1.3/api/rest", + Username: "admin", + Password: "pass", + BlockProtocol: identifiers.ISCSITransport, + Insecure: true, + Client: clientMock, + IP: "192.168.1.3", + NASCooldownTracker: array.NewNASCooldown(time.Minute, 5), + } + arrays := ctrlSvc.Arrays() + arrays["APM0012345"] = metroArr + arrays["APM0045678"] = metroRemoteArr + ctrlSvc.SetArrays(arrays) + }) + + ginkgo.It("should pass with node selector passing [Block]", func() { + // Create a node that matches the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "zone1", // matches the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + k8sutils.GetNodeByCSINodeID = func(_ context.Context, _ string, _ string, _ string) (*k8score.Node, error) { + return node, nil + } + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") + + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) + }) + + ginkgo.It("should pass with local node selector passing [Block]", func() { + // Create a node that matches the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "nonu1", // matches the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + k8sutils.GetNodeByCSINodeID = func(_ context.Context, _ string, _ string, _ string) (*k8score.Node, error) { + return node, nil + } + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") + + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) + }) + ginkgo.It("should pass with local node selector of remote array passing [Block]", func() { + // Create a node that matches the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "nonu2", // matches the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + k8sutils.GetNodeByCSINodeID = func(_ context.Context, _ string, _ string, _ string) (*k8score.Node, error) { + return node, nil + } + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") + + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) }) - ginkgo.It("should fail", func() { - ip := "127.0.0.1" // we don't have array with this IP - volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, ip) + ginkgo.It("should fail due to node selector mismatch [Block]", func() { + // Create a node that does not match the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "zone5", // does not match the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + k8sutils.GetNodeByCSINodeID = func(_ context.Context, _ string, _ string, _ string) (*k8score.Node, error) { + return node, nil + } + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") + + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.HaveOccurred()) + }) + + ginkgo.It("should fail due to no node presence [Block]", func() { + k8sutils.GetNodeByCSINodeID = func(_ context.Context, _ string, _ string, _ string) (*k8score.Node, error) { + return nil, fmt.Errorf("failed to list nodes") + } + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") + + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.HaveOccurred()) + }) + ginkgo.It("should fail due to no remote system [Block]", func() { + // Create a node that matches the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "zone1", // matches the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + k8sutils.GetNodeByCSINodeID = func(_ context.Context, _ string, _ string, _ string) (*k8score.Node, error) { + return node, nil + } + clientMock.On("GetVolume", mock.Anything, validRemoteVolID+"1"). + Return(gopowerstore.Volume{}, fmt.Errorf("failed to get volume")) + // Invalid remote system + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID+"1", "APM0456789") + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err).To(gomega.HaveOccurred()) gomega.Expect(err.Error()).To(gomega.ContainSubstring("cannot find remote array")) }) }) @@ -4139,7 +5617,7 @@ var _ = ginkgo.Describe("CSIControllerService", func() { }) clientMock.On("GetFS", mock.Anything, validBaseVolID). - Return(gopowerstore.FileSystem{}, nil).Once() + Return(gopowerstore.FileSystem{ID: validBaseVolID}, nil) }) exportID := "some-export-id" @@ -4155,7 +5633,7 @@ var _ = ginkgo.Describe("CSIControllerService", func() { clientMock.On("ModifyNFSExport", mock.Anything, mock.Anything, exportID).Return(gopowerstore.CreateResponse{}, nil) - req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validBaseVolID, NodeId: validNodeID} + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validNfsVolumeID, NodeId: validNodeID} res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) gomega.Expect(err).To(gomega.BeNil()) @@ -4180,10 +5658,9 @@ var _ = ginkgo.Describe("CSIControllerService", func() { }).Once() }) - req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validBaseVolID, NodeId: validNodeID} + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validNfsVolumeID, NodeId: validNodeID} res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) }) @@ -4201,6 +5678,9 @@ var _ = ginkgo.Describe("CSIControllerService", func() { }, }).Once() + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{}, nil) + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validBlockVolumeID, NodeId: nodeID} _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) @@ -4223,6 +5703,8 @@ var _ = ginkgo.Describe("CSIControllerService", func() { ginkgo.When("host does not exist", func() { ginkgo.It("should fail", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{}, nil) clientMock.On("GetHostByName", mock.Anything, validNodeID). Return(gopowerstore.Host{}, gopowerstore.APIError{ ErrorMsg: &api.ErrorMsg{ @@ -4248,6 +5730,9 @@ var _ = ginkgo.Describe("CSIControllerService", func() { ginkgo.When("fail to check host", func() { ginkgo.It("should fail", func() { e := errors.New("some-api-error") + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{}, nil) + clientMock.On("GetHostByName", mock.Anything, validNodeID). Return(gopowerstore.Host{}, e).Once() @@ -4303,181 +5788,182 @@ var _ = ginkgo.Describe("CSIControllerService", func() { }) ginkgo.Describe("calling ListVolumes()", func() { + clientA := &gopowerstoremock.Client{} + clientB := &gopowerstoremock.Client{} + + injectArrays := func() { + arrMap := map[string]*array.PowerStoreArray{ + "globalvolid1": {Client: clientA, GlobalID: "globalvolid1"}, + "globalvolid2": {Client: clientB, GlobalID: "globalvolid2"}, + } + ctrlSvc.SetArrays(arrMap) + } + mockCalls := func() { - clientMock.On("GetVolumes", mock.Anything). - Return([]gopowerstore.Volume{ - { - ID: "arr1-id1", - Name: "arr1-vol1", - }, - { - ID: "arr1-id2", - Name: "arr1-vol2", - }, - }, nil).Once() - clientMock.On("GetVolumes", mock.Anything). + // --- Array globalvolid1 (clientA) --- + clientA.On("GetHostVolumeMappings", mock.Anything). + Return([]gopowerstore.HostVolumeMapping{}, nil).Once() + + clientA.On("GetVolumes", mock.Anything). Return([]gopowerstore.Volume{ - { - ID: "arr2-id1", - Name: "arr2-vol1", - }, + {ID: "arr1-id1", Name: "arr1-vol1"}, + {ID: "arr1-id2", Name: "arr1-vol2"}, }, nil).Once() - clientMock.On("ListFS", mock.Anything). + + clientA.On("ListFS", mock.Anything). Return([]gopowerstore.FileSystem{ - { - ID: "arr3-id1", - Name: "arr3-fs1", - }, - { - ID: "arr3-id2", - Name: "arr3-fs2", - }, + {ID: "arr3-id1", Name: "arr3-fs1"}, + {ID: "arr3-id2", Name: "arr3-fs2"}, + }, nil).Once() + + // --- Array globalvolid2 (clientB) --- + clientB.On("GetHostVolumeMappings", mock.Anything). + Return([]gopowerstore.HostVolumeMapping{}, nil).Once() + + clientB.On("GetVolumes", mock.Anything). + Return([]gopowerstore.Volume{ + {ID: "arr2-id1", Name: "arr2-vol1"}, }, nil).Once() - clientMock.On("ListFS", mock.Anything). + + clientB.On("ListFS", mock.Anything). Return([]gopowerstore.FileSystem{ - { - ID: "arr4-id1", - Name: "arr4-fs1", - }, + {ID: "arr4-id1", Name: "arr4-fs1"}, }, nil).Once() } + // helper to assert the response contains exactly the expected set of volume IDs (order-independent) + expectContainsVolumeIDs := func(res *csi.ListVolumesResponse, expected []string) { + gomega.Expect(res).ToNot(gomega.BeNil()) + got := make(map[string]struct{}, len(res.Entries)) + for _, e := range res.Entries { + if e != nil && e.Volume != nil { + got[e.Volume.VolumeId] = struct{}{} + } + } + for _, id := range expected { + _, ok := got[id] + gomega.Expect(ok).To(gomega.BeTrue(), "missing expected volumeID: %s; got=%v", id, got) + } + // exact-match check + gomega.Expect(len(got)).To(gomega.Equal(len(expected))) + } + ginkgo.When("there is no parameters", func() { - ginkgo.It("should return all volumes from both arrays", func() { + ginkgo.It("should return all volumes from both arrays (order-independent)", func() { + injectArrays() mockCalls() req := &csi.ListVolumesRequest{} res, err := ctrlSvc.ListVolumes(context.Background(), req) - - gomega.Expect(res).To(gomega.Equal(&csi.ListVolumesResponse{ - Entries: []*csi.ListVolumesResponse_Entry{ - { - Volume: &csi.Volume{ - VolumeId: "arr1-id1", - }, - }, - { - Volume: &csi.Volume{ - VolumeId: "arr1-id2", - }, - }, - { - Volume: &csi.Volume{ - VolumeId: "arr2-id1", - }, - }, - { - Volume: &csi.Volume{ - VolumeId: "arr3-id1", - }, - }, - { - Volume: &csi.Volume{ - VolumeId: "arr3-id2", - }, - }, - { - Volume: &csi.Volume{ - VolumeId: "arr4-id1", - }, - }, - }, - })) gomega.Expect(err).To(gomega.BeNil()) + + expected := []string{ + "arr1-id1/globalvolid1/scsi", + "arr1-id2/globalvolid1/scsi", + "arr2-id1/globalvolid2/scsi", + "arr3-id1/globalvolid1/nfs", + "arr3-id2/globalvolid1/nfs", + "arr4-id1/globalvolid2/nfs", + } + expectContainsVolumeIDs(res, expected) }) }) ginkgo.When("passing max entries", func() { - ginkgo.It("should return 'n' entries and next token", func() { + ginkgo.It("should return 'n' entries and a next token (when applicable)", func() { + injectArrays() mockCalls() - req := &csi.ListVolumesRequest{ - MaxEntries: 1, - } + req := &csi.ListVolumesRequest{MaxEntries: 1} res, err := ctrlSvc.ListVolumes(context.Background(), req) - gomega.Expect(res).To(gomega.Equal(&csi.ListVolumesResponse{ - Entries: []*csi.ListVolumesResponse_Entry{ - { - Volume: &csi.Volume{ - VolumeId: "arr1-id1", - }, - }, - }, - NextToken: "1", - })) gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).ToNot(gomega.BeNil()) + gomega.Expect(len(res.Entries)).To(gomega.Equal(1)) + gomega.Expect(res.NextToken).To(gomega.Satisfy(func(token string) bool { return token != "" })) }) }) - ginkgo.When("using next token", func() { - ginkgo.It("should return volumes starting from token", func() { + ginkgo.When("using starting token", func() { + ginkgo.It("should return volumes starting from token (len check)", func() { + injectArrays() mockCalls() - req := &csi.ListVolumesRequest{ - MaxEntries: 1, - StartingToken: "1", - } + req := &csi.ListVolumesRequest{MaxEntries: 1, StartingToken: "1"} res, err := ctrlSvc.ListVolumes(context.Background(), req) - gomega.Expect(res).To(gomega.Equal(&csi.ListVolumesResponse{ - Entries: []*csi.ListVolumesResponse_Entry{ - { - Volume: &csi.Volume{ - VolumeId: "arr1-id2", - }, - }, - }, - NextToken: "2", - })) gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).ToNot(gomega.BeNil()) + gomega.Expect(len(res.Entries)).To(gomega.Equal(1)) + + expectedAll := []string{ + "arr1-id1/globalvolid1/scsi", + "arr1-id2/globalvolid1/scsi", + "arr2-id1/globalvolid2/scsi", + "arr3-id1/globalvolid1/nfs", + "arr3-id2/globalvolid1/nfs", + "arr4-id1/globalvolid2/nfs", + } + gotID := res.Entries[0].Volume.VolumeId + found := false + for _, e := range expectedAll { + if e == gotID { + found = true + break + } + } + gomega.Expect(found).To(gomega.BeTrue(), "returned unexpected volume id: %s", gotID) }) }) ginkgo.When("using wrong token", func() { ginkgo.It("should fail [not parsable]", func() { + injectArrays() token := "as!512$25%!_" // #nosec G101 - req := &csi.ListVolumesRequest{ - MaxEntries: 1, - StartingToken: token, - } + req := &csi.ListVolumesRequest{MaxEntries: 1, StartingToken: token} res, err := ctrlSvc.ListVolumes(context.Background(), req) gomega.Expect(res).To(gomega.BeNil()) gomega.Expect(err).ToNot(gomega.BeNil()) - gomega.Expect(err.Error()).To(gomega.ContainSubstring("unable to parse StartingToken: %v into uint32", token)) + // check parse error text exists + gomega.Expect(err.Error()).To(gomega.ContainSubstring("unable to parse StartingToken")) }) - ginkgo.It("shoud fail [too high]", func() { - tokenInt := 200 - token := "200" - + ginkgo.It("should fail [too high]", func() { + injectArrays() mockCalls() - req := &csi.ListVolumesRequest{ - MaxEntries: 1, - StartingToken: token, - } + req := &csi.ListVolumesRequest{MaxEntries: 1, StartingToken: "200"} res, err := ctrlSvc.ListVolumes(context.Background(), req) gomega.Expect(res).To(gomega.BeNil()) gomega.Expect(err).ToNot(gomega.BeNil()) - gomega.Expect(err.Error()).To(gomega.ContainSubstring("startingToken=%d > len(volumes)=%d", tokenInt, 6)) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("startingToken=")) }) }) ginkgo.When("get volumes return error", func() { - ginkgo.It("should fail]", func() { - clientMock.On("GetVolumes", mock.Anything). - Return([]gopowerstore.Volume{}, gopowerstore.NewNotFoundError()) - clientMock.On("ListFS", mock.Anything). - Return([]gopowerstore.FileSystem{}, gopowerstore.NewNotFoundError()) + ginkgo.It("should fail when backing client returns an error", func() { + injectArrays() + // simulate failures for both arrays' calls + clientA.On("GetVolumes", mock.Anything). + Return([]gopowerstore.Volume{}, gopowerstore.NewNotFoundError()).Once() + clientA.On("ListFS", mock.Anything). + Return([]gopowerstore.FileSystem{}, gopowerstore.NewNotFoundError()).Once() + clientA.On("GetHostVolumeMappings", mock.Anything). + Return([]gopowerstore.HostVolumeMapping{}, gopowerstore.NewNotFoundError()).Once() + + clientB.On("GetVolumes", mock.Anything). + Return([]gopowerstore.Volume{}, gopowerstore.NewNotFoundError()).Once() + clientB.On("ListFS", mock.Anything). + Return([]gopowerstore.FileSystem{}, gopowerstore.NewNotFoundError()).Once() + clientB.On("GetHostVolumeMappings", mock.Anything). + Return([]gopowerstore.HostVolumeMapping{}, gopowerstore.NewNotFoundError()).Once() + req := &csi.ListVolumesRequest{} res, err := ctrlSvc.ListVolumes(context.Background(), req) - gomega.Expect(res).To(gomega.BeNil()) gomega.Expect(err).ToNot(gomega.BeNil()) - gomega.Expect(err.Error()).To(gomega.ContainSubstring("unable to list volumes")) }) }) }) @@ -5413,7 +6899,8 @@ var _ = ginkgo.Describe("CSIControllerService", func() { ginkgo.It("should fail if volume is single", func() { clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). Return(gopowerstore.VolumeGroups{}, gopowerstore.APIError{}) - + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) req := &csiext.CreateStorageProtectionGroupRequest{ VolumeHandle: validBaseVolID + "/" + firstValidID + "/" + "iscsi", } @@ -5431,6 +6918,8 @@ var _ = ginkgo.Describe("CSIControllerService", func() { clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, validGroupID). Return(gopowerstore.ReplicationSession{}, gopowerstore.APIError{}) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) req := &csiext.CreateStorageProtectionGroupRequest{ VolumeHandle: validBaseVolID + "/" + firstValidID + "/" + "iscsi", @@ -5584,9 +7073,220 @@ var _ = ginkgo.Describe("CSIControllerService", func() { "replication of volumes that aren't assigned to group is not implemented yet", )) }) + ginkgo.It("should fail if cluster retrieval fails", func() { + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). + Return(gopowerstore.VolumeGroups{VolumeGroup: []gopowerstore.VolumeGroup{{ID: validGroupID}}}, nil) + + clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, validGroupID). + Return(gopowerstore.ReplicationSession{ + LocalResourceID: validGroupID, + RemoteResourceID: validRemoteGroupID, + RemoteSystemID: validRemoteSystemID, + StorageElementPairs: []gopowerstore.StorageElementPair{{ + LocalStorageElementID: validBaseVolID, + RemoteStorageElementID: validRemoteVolID, + }}, + }, nil) + + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Size: validVolSize}, nil) + + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{}, errors.New("cluster not found")) + + req := &csiext.CreateRemoteVolumeRequest{ + VolumeHandle: validBaseVolID + "/" + firstValidID + "/" + "iscsi", + } + res, err := ctrlSvc.CreateRemoteVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("cluster not found")) + }) + ginkgo.It("should return info if everything is ok for NFS", func() { + clientMock.On("GetFS", mock.Anything, validBaseVolID).Return(gopowerstore.FileSystem{ID: validBaseVolID}, nil) + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID).Return(gopowerstore.NFSExport{ID: validNasID}, nil) + + clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, mock.AnythingOfType("string")). + Return(gopowerstore.ReplicationSession{ + LocalResourceID: validNfsVolumeID, + RemoteResourceID: validRemoteGroupID, + RemoteSystemID: validRemoteSystemID, + StorageElementPairs: []gopowerstore.StorageElementPair{ + { + LocalStorageElementID: validBaseVolID, + RemoteStorageElementID: validRemoteVolID, + }, + }, + }, nil) + + clientMock.On("GetCluster", mock.Anything).Return(gopowerstore.Cluster{Name: validClusterName}, nil) + + clientMock.On("GetRemoteSystem", mock.Anything, validRemoteSystemID).Return(gopowerstore.RemoteSystem{ + Name: validRemoteSystemName, + ManagementAddress: secondValidID, + ID: validRemoteSystemID, + SerialNumber: validRemoteSystemGlobalID, + }, nil) + + req := &csiext.CreateRemoteVolumeRequest{ + VolumeHandle: validBaseVolID + "/" + firstValidID + "/" + "nfs", + } + + res, err := ctrlSvc.CreateRemoteVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csiext.CreateRemoteVolumeResponse{RemoteVolume: &csiext.Volume{ + CapacityBytes: 0, + VolumeId: validBaseVolID + "/" + validRemoteSystemGlobalID + "/" + "nfs", + VolumeContext: map[string]string{ + "remoteSystem": validClusterName, + "managementAddress": secondValidID, + "arrayID": validRemoteSystemGlobalID, + }, + }})) + }) + ginkgo.It("should handle error when NFS export retrieval fails and create a new export", func() { + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). + Return(gopowerstore.NFSExport{}, errors.New("NFS export not found")).Once() + + clientMock.On("GetFS", mock.Anything, validBaseVolID). + Return(gopowerstore.FileSystem{ID: validBaseVolID, Name: "fs_name"}, nil) + + clientMock.On("CreateNFSExport", mock.Anything, mock.MatchedBy(func(req *gopowerstore.NFSExportCreate) bool { + return req != nil && + req.FileSystemID == validBaseVolID && + req.Name == "auto_export_"+validBaseVolID[:8] && + req.Path == "/fs_name" + })).Return(gopowerstore.CreateResponse{ID: "new-export-id"}, nil) + + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). + Return(gopowerstore.NFSExport{ID: "new-export-id"}, nil).Once() + clientMock.On("GetCluster", mock.Anything).Return(gopowerstore.Cluster{Name: validClusterName}, nil) + + clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, validBaseVolID). + Return(gopowerstore.ReplicationSession{ + RemoteSystemID: validRemoteSystemID, + LocalResourceID: validBaseVolID, + RemoteResourceID: validRemoteGroupID, + }, nil) + + clientMock.On("GetRemoteSystem", mock.Anything, validRemoteSystemID). + Return(gopowerstore.RemoteSystem{ + Name: "remote-system", + ManagementAddress: "10.0.0.2", + SerialNumber: "RSN-999", + }, nil) + + arr := &array.PowerStoreArray{Client: clientMock} + // reuse the existing test controller and inject the test array into its Arrays map + arrays := ctrlSvc.Arrays() + arrays["array-id"] = arr + ctrlSvc.SetArrays(arrays) + + req := &csiext.CreateRemoteVolumeRequest{ + VolumeHandle: validBaseVolID + "/array-id/nfs", + } + + res, err := ctrlSvc.CreateRemoteVolume(context.TODO(), req) + + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Expect(res).ToNot(gomega.BeNil()) + gomega.Expect(res.GetRemoteVolume().GetVolumeContext()["replication.NasServerID"]).To(gomega.Equal("")) + }) + ginkgo.It("should return error when CreateNFSExport fails", func() { + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). + Return(gopowerstore.NFSExport{}, gopowerstore.NewNotFoundError()).Once() + + clientMock.On("GetFS", mock.Anything, validBaseVolID). + Return(gopowerstore.FileSystem{ID: validBaseVolID, Name: "fs_name"}, nil) + + clientMock.On("CreateNFSExport", mock.Anything, mock.AnythingOfType("*gopowerstore.NFSExportCreate")). + Return(gopowerstore.CreateResponse{}, errors.New("mock export creation failure")) + + arr := &array.PowerStoreArray{Client: clientMock} + // reuse the existing test controller and inject the test array into its Arrays map + arrays := ctrlSvc.Arrays() + arrays["array-id"] = arr + ctrlSvc.SetArrays(arrays) + controller := ctrlSvc + + req := &csiext.CreateRemoteVolumeRequest{ + VolumeHandle: validBaseVolID + "/array-id/nfs", + } + + res, err := controller.CreateRemoteVolume(context.TODO(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(status.Code(err)).To(gomega.Equal(codes.Internal)) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to create NFS export")) + }) + ginkgo.It("should return error when replication session is not found", func() { + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). + Return(gopowerstore.NFSExport{ID: "export-id"}, nil) + + clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, validBaseVolID). + Return(gopowerstore.ReplicationSession{}, errors.New("replication session not found")) + + arr := &array.PowerStoreArray{Client: clientMock} + arrays := ctrlSvc.Arrays() + arrays["array-id"] = arr + ctrlSvc.SetArrays(arrays) + controller := ctrlSvc + + req := &csiext.CreateRemoteVolumeRequest{ + VolumeHandle: validBaseVolID + "/array-id/nfs", + } + + res, err := controller.CreateRemoteVolume(context.TODO(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(status.Code(err)).To(gomega.Equal(codes.Internal)) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("no replication session found")) + }) + ginkgo.It("should create storage protection group for NFS volume", func() { + // Setup mocks + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). + Return(gopowerstore.NFSExport{ID: validNasID}, nil) + + clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, validBaseVolID). + Return(gopowerstore.ReplicationSession{ + LocalResourceID: validBaseVolID, + RemoteResourceID: validRemoteGroupID, + RemoteSystemID: validRemoteSystemID, + StorageElementPairs: []gopowerstore.StorageElementPair{ + { + LocalStorageElementID: validBaseVolID, + RemoteStorageElementID: validRemoteVolID, + }, + }, + }, nil) + + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) + + clientMock.On("GetRemoteSystem", mock.Anything, validRemoteSystemID). + Return(gopowerstore.RemoteSystem{ + Name: validRemoteSystemName, + ManagementAddress: secondValidID, + ID: validRemoteSystemID, + SerialNumber: validRemoteSystemGlobalID, + }, nil) + + req := &csiext.CreateStorageProtectionGroupRequest{ + VolumeHandle: validBaseVolID + "/" + firstValidID + "/nfs", + } + + res, err := ctrlSvc.CreateStorageProtectionGroup(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.LocalProtectionGroupAttributes).To(gomega.HaveKeyWithValue("systemName", "localSystemName")) + gomega.Expect(res.RemoteProtectionGroupAttributes).To(gomega.HaveKeyWithValue("remoteSystemName", "localSystemName")) + }) }) }) - ginkgo.Describe("calling EnsureProtectionPolicyExists", func() { ginkgo.When("ensure protection policy exists", func() { ginkgo.It("should failed if remote system not in list", func() { diff --git a/pkg/controller/creator.go b/pkg/controller/creator.go index 08f3df5e..a7c5b9cf 100644 --- a/pkg/controller/creator.go +++ b/pkg/controller/creator.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,9 +26,8 @@ import ( "github.com/dell/csi-powerstore/v2/pkg/identifiers" "github.com/dell/gopowerstore/api" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/gopowerstore" - log "github.com/sirupsen/logrus" + "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -83,7 +82,7 @@ func setMetaData(reqParams map[string]string, createParams interface{}) { t.MetaData().Set(HeaderPersistentVolumeClaimName, reqParams[CSIPersistentVolumeClaimName]) t.MetaData().Set(HeaderPersistentVolumeClaimNamespace, reqParams[CSIPersistentVolumeClaimNamespace]) } else { - log.Printf("warning: %T: no MetaData method exists, consider updating gopowerstore library.", createParams) + log.Infof("warning: %T: no MetaData method exists, consider updating gopowerstore library.", createParams) } } @@ -373,7 +372,8 @@ type NfsCreator struct { } // CheckSize validates that size is correct and returns size in bytes -func (*NfsCreator) CheckSize(_ context.Context, cr *csi.CapacityRange, isAutoRoundOffFsSizeEnabled bool) (int64, error) { +func (*NfsCreator) CheckSize(ctx context.Context, cr *csi.CapacityRange, isAutoRoundOffFsSizeEnabled bool) (int64, error) { + log := log.WithContext(ctx) minSize := cr.GetRequiredBytes() maxSize := cr.GetLimitBytes() @@ -409,6 +409,7 @@ func (*NfsCreator) CheckName(_ context.Context, name string) error { // CheckIfAlreadyExists queries storage array if FileSystem with given name exists func (c *NfsCreator) CheckIfAlreadyExists(ctx context.Context, name string, sizeInBytes int64, client gopowerstore.Client) (*csi.Volume, error) { + log := log.WithContext(ctx) alreadyExistVolume, err := client.GetFSByName(ctx, name) if err != nil { return nil, status.Errorf(status.Code(err), "can't find filesystem '%s': %s", name, err.Error()) diff --git a/pkg/controller/creator_test.go b/pkg/controller/creator_test.go index 730b8e98..933d7e6d 100644 --- a/pkg/controller/creator_test.go +++ b/pkg/controller/creator_test.go @@ -24,9 +24,9 @@ import ( "strings" "testing" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/gopowerstore" "github.com/dell/gopowerstore/mocks" + "github.com/container-storage-interface/spec/lib/go/csi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) diff --git a/pkg/controller/csi_extension_server.go b/pkg/controller/csi_extension_server.go index 3ed0f390..592538c7 100644 --- a/pkg/controller/csi_extension_server.go +++ b/pkg/controller/csi_extension_server.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2022-2024 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2022-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,7 +29,6 @@ import ( vgsext "github.com/dell/dell-csi-extensions/volumeGroupSnapshot" "github.com/dell/gopowerstore" "github.com/go-openapi/strfmt" - log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -39,6 +38,7 @@ const StateReady = "Ready" // CreateVolumeGroupSnapshot creates volume group snapshot func (s *Service) CreateVolumeGroupSnapshot(ctx context.Context, request *vgsext.CreateVolumeGroupSnapshotRequest) (*vgsext.CreateVolumeGroupSnapshotResponse, error) { + log := log.WithContext(ctx) log.Infof("CreateVolumeGroupSnapshot called with req: %v", request) err := validateCreateVGSreq(request) @@ -157,20 +157,17 @@ func (s *Service) CreateVolumeGroupSnapshot(ctx context.Context, request *vgsext func validateCreateVGSreq(request *vgsext.CreateVolumeGroupSnapshotRequest) error { if request.Name == "" { err := status.Error(codes.InvalidArgument, "CreateVolumeGroupSnapshotRequest needs Name to be set") - log.Errorf("Error from validateCreateVGSreq: %v ", err) return err } // name must be less than 28 chars, because we name snapshots with -, and index can at most be 3 chars if len(request.Name) > 27 { err := status.Errorf(codes.InvalidArgument, "Requested name %s longer than 27 character max", request.Name) - log.Errorf("Error from validateCreateVGSreq: %v ", err) return err } if len(request.SourceVolumeIDs) == 0 { err := status.Errorf(codes.InvalidArgument, "Source volumes are not present") - log.Errorf("Error from validateCreateVGSreq: %v ", err) return err } @@ -179,7 +176,7 @@ func validateCreateVGSreq(request *vgsext.CreateVolumeGroupSnapshotRequest) erro // ValidateVolumeHostConnectivity menthod will be called by podmon sidecars to check host connectivity with array func (s *Service) ValidateVolumeHostConnectivity(ctx context.Context, req *podmon.ValidateVolumeHostConnectivityRequest) (*podmon.ValidateVolumeHostConnectivityResponse, error) { - // ctx, log, _ := GetRunIDLog(ctx) + log := log.WithContext(ctx) log.Infof("ValidateVolumeHostConnectivity called %+v", req) rep := &podmon.ValidateVolumeHostConnectivityResponse{ Messages: make([]string, 0), @@ -200,17 +197,25 @@ func (s *Service) ValidateVolumeHostConnectivity(ctx context.Context, req *podmo if globalID == "" { if len(req.GetVolumeIds()) == 0 { log.Info("neither globalId nor volumeID is present in request") - globalIDs[s.DefaultArray().GlobalID] = true + // need to put all arrays to check not only default array and not matched ID will be filtered later. + for _, array := range s.Arrays() { + globalIDs[array.GlobalID] = true + } } - // for loop req.GetVolumeIds() + for _, volID := range req.GetVolumeIds() { volumeHandle, err := array.ParseVolumeID(ctx, volID, s.DefaultArray(), nil) - globalID = volumeHandle.LocalArrayGlobalID - if err != nil || globalID == "" { + if err != nil || (volumeHandle.LocalArrayGlobalID == "" && volumeHandle.RemoteArrayGlobalID == "") { log.Errorf("unable to retrieve array's globalID after parsing volumeID") globalIDs[s.DefaultArray().GlobalID] = true } else { - globalIDs[globalID] = true + if volumeHandle.LocalArrayGlobalID != "" { + globalIDs[volumeHandle.LocalArrayGlobalID] = true + } + + if volumeHandle.RemoteArrayGlobalID != "" { + globalIDs[volumeHandle.RemoteArrayGlobalID] = true + } } } } else { @@ -219,13 +224,24 @@ func (s *Service) ValidateVolumeHostConnectivity(ctx context.Context, req *podmo // Go through each of the globalIDs for globalID := range globalIDs { + // Check if array is non-uniform and matches the node label + arr, err := s.GetOneArray(globalID) + if err != nil || arr == nil { + log.Errorf("failed to get array %s: %s", globalID, err.Error()) + return nil, err + } + + if !arr.CheckConnectivity(ctx, req.GetNodeId()) { + log.Warnf("Not a match for node %s on array %s, skipping connectivity check", req.GetNodeId(), globalID) + continue + } + // First - check if the array is visible from the node - err := s.checkIfNodeIsConnected(ctx, globalID, req.GetNodeId(), rep) + err = s.checkIfNodeIsConnected(ctx, globalID, req.GetNodeId(), rep) if err != nil { return rep, err } } - // Check for IOinProgress only when volumes IDs are present in the request as the field is required only in the latter case also to reduce number of calls to the API making it efficient if len(req.GetVolumeIds()) > 0 { // Get array config @@ -257,7 +273,8 @@ func (s *Service) ValidateVolumeHostConnectivity(ctx context.Context, req *podmo // This context is for the set of requests for the current volume. // Used to cancel any pending requests before checking for IO on any // subsequent volumes. - ioCtx, ioCtxCancel := context.WithCancel(ctx) + probeBase := context.WithoutCancel(ctx) // decouple from caller's cancel/deadline + ioCtx, ioCtxCancel := context.WithTimeout(probeBase, identifiers.PodmonArrayConnectivityTimeout) // channels for receiving responses from async requests reqChs := make([]<-chan error, 0) @@ -305,6 +322,7 @@ func waitAndClose(wg *sync.WaitGroup, ch chan error) { // fan-in concurrency pattern and returns true if at least one response is a nil error, // denoting IO is in-progress. func isIOInProgress(ctx context.Context, chs ...<-chan error) bool { + log := log.WithContext(ctx) // single channel on which the channels in "chs" will write their results errCh := make(chan error) wg := &sync.WaitGroup{} @@ -366,6 +384,7 @@ func isIOInProgress(ctx context.Context, chs ...<-chan error) bool { // It can be used to dispatch multiple requests in parallel for situations such as metro // volumes where multiple volumes need to be checked for IO to determine if the volume is active. func asyncGetIOInProgress(ctx context.Context, volID string, array array.PowerStoreArray, protocol string) <-chan error { + log := log.WithContext(ctx) errCh := make(chan error) go func() { defer close(errCh) @@ -388,6 +407,7 @@ func asyncGetIOInProgress(ctx context.Context, volID string, array array.PowerSt // checkIfNodeIsConnected looks at the 'nodeId' to determine if there is connectivity to the 'arrayId' array. // The 'rep' object will be filled with the results of the check. func (s *Service) checkIfNodeIsConnected(ctx context.Context, arrayID string, nodeID string, rep *podmon.ValidateVolumeHostConnectivityResponse) error { + log := log.WithContext(ctx) log.Infof("Checking if array %s is connected to node %s", arrayID, nodeID) var message string rep.Connected = false @@ -422,6 +442,7 @@ func (s *Service) checkIfNodeIsConnected(ctx context.Context, arrayID string, no // getIOInProgress attempts to determine if IO has recently occurred for a given volume, volID, // and returns a nil error if IO has occurred. func getIOInProgress(ctx context.Context, volID string, arrayConfig array.PowerStoreArray, protocol string) (err error) { + log := log.WithContext(ctx) // Call PerformanceMetricsByVolume or PerformanceMetricsByFileSystem in gopowerstore based on the volume type if protocol == "scsi" { resp, err := arrayConfig.Client.PerformanceMetricsByVolume(ctx, volID, gopowerstore.TwentySec) diff --git a/pkg/controller/csi_extension_server_test.go b/pkg/controller/csi_extension_server_test.go index 58f422f9..ddb3928b 100644 --- a/pkg/controller/csi_extension_server_test.go +++ b/pkg/controller/csi_extension_server_test.go @@ -42,6 +42,7 @@ import ( gomega "github.com/onsi/gomega" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + k8score "k8s.io/api/core/v1" ) const ( @@ -373,6 +374,52 @@ var _ = ginkgo.Describe("csi-extension-server", func() { gomega.Expect(response.IosInProgress).To(gomega.BeTrue()) }) }) + + ginkgo.It("unable to parse volume ID - defaults to array - volumeHandle is empty", func() { + req := &podmon.ValidateVolumeHostConnectivityRequest{ + ArrayId: "", + VolumeIds: []string{""}, + NodeId: validNodeID, + } + + res, err := ctrlSvc.ValidateVolumeHostConnectivity(context.Background(), req) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + }) + + ginkgo.It("No matching host connectivity found and no volume ID in request - empty response", func() { + // Set the host connectivity for the first array when the node does not match for labels. + arr := ctrlSvc.Arrays() + gomega.Expect(arr).ToNot(gomega.BeNil()) + + arr[firstValidID].HostConnectivity = &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "zone1", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"local"}, + }, + }, + }, + }, + }, + }, + } + ctrlSvc.SetArrays(arr) + + req := &podmon.ValidateVolumeHostConnectivityRequest{ + ArrayId: firstValidID, + NodeId: validNodeID, + } + + res, err := ctrlSvc.ValidateVolumeHostConnectivity(context.Background(), req) + gomega.Expect(res).ToNot(gomega.BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + }) }) ginkgo.Describe("calling IsIOInProgress and QueryArrayStatus", func() { diff --git a/pkg/controller/publisher.go b/pkg/controller/publisher.go index ee884694..5b544407 100644 --- a/pkg/controller/publisher.go +++ b/pkg/controller/publisher.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2024 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,10 +24,9 @@ import ( "strconv" "strings" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/pkg/identifiers" "github.com/dell/gopowerstore" - log "github.com/sirupsen/logrus" + "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -48,6 +47,7 @@ type SCSIPublisher struct{} func (s *SCSIPublisher) Publish(ctx context.Context, publishContext map[string]string, req *csi.ControllerPublishVolumeRequest, client gopowerstore.Client, kubeNodeID string, volumeID string, isRemote bool, ) (*csi.ControllerPublishVolumeResponse, error) { + log := log.WithContext(ctx) volume, err := client.GetVolume(ctx, volumeID) if err != nil { if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { @@ -171,12 +171,16 @@ func (s *SCSIPublisher) addLUNIDToPublishContext( type NfsPublisher struct { // ExternalAccess used to set custom ip to be added to the NFS Export 'hosts' list ExternalAccess string + + // ExclusiveAccess indicates whether only externalAccess entries should be added to the NFS export + ExclusiveAccess bool } // Publish publishes FileSystem by adding host (node) to the NFS Export 'hosts' list func (n *NfsPublisher) Publish(ctx context.Context, publishContext map[string]string, req *csi.ControllerPublishVolumeRequest, client gopowerstore.Client, kubeNodeID string, volumeID string, _ bool, ) (*csi.ControllerPublishVolumeResponse, error) { + log := log.WithContext(ctx) fs, err := client.GetFS(ctx, volumeID) if err != nil { if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { @@ -192,7 +196,6 @@ func (n *NfsPublisher) Publish(ctx context.Context, publishContext map[string]st ip := ipList[0] ipWithNat := make([]string, 0, 2) - ipWithNat = append(ipWithNat, ip) // Create NFS export if it doesn't exist _, err = client.GetNFSExportByFileSystemID(ctx, fs.ID) @@ -217,12 +220,22 @@ func (n *NfsPublisher) Publish(ctx context.Context, publishContext map[string]st return nil, status.Errorf(codes.Internal, "failure getting nfs export: %s", err.Error()) } + // Add host IP to nfs export if not already present + if !identifiers.HostAlreadyPresentInNFSExport(export, ip) { + log.Debug("IPs have not been added") + + if !n.ExclusiveAccess { + log.Infof("Exclusive access is disabled, adding nodeIP %s to the nfs export", ip) + ipWithNat = append(ipWithNat, ip) + } + } + if n.ExternalAccess != "" && !identifiers.ExternalAccessAlreadyAdded(export, n.ExternalAccess) { externalAccess, err := identifiers.GetIPListWithMaskFromString(n.ExternalAccess) if err != nil { return nil, status.Errorf(codes.InvalidArgument, "can't find IP in X_CSI_POWERSTORE_EXTERNAL_ACCESS variable") } - log.Debug("externalAccess parsed IP:", externalAccess) + log.Debugf("externalAccess parsed IP: %s", externalAccess) ipWithNat = append(ipWithNat, externalAccess) } // Add host IP to existing nfs export @@ -230,7 +243,7 @@ func (n *NfsPublisher) Publish(ctx context.Context, publishContext map[string]st AddRWRootHosts: ipWithNat, }, export.ID) if err != nil { - log.Debug("Error while PublishVolume: ", err.Error()) + log.Debugf("Error while PublishVolume: %s ", err.Error()) if apiError, ok := err.(gopowerstore.APIError); !(ok && (apiError.NotFound() || apiError.HostAlreadyPresentInNFSExport())) { return nil, status.Errorf(codes.Internal, "failure when adding new host to nfs export: %s", err.Error()) } @@ -246,7 +259,16 @@ func (n *NfsPublisher) Publish(ctx context.Context, publishContext map[string]st } publishContext[KeyNasName] = nas.Name // we need to pass that to node part of the driver publishContext[identifiers.KeyNfsExportPath] = fileInterface.IPAddress + ":/" + export.Name - publishContext[identifiers.KeyHostIP] = ipWithNat[0] + + // Add node IP to publish context only if exclusive access is not set + if !n.ExclusiveAccess { + // Ensure we don't index an empty slice; fallback to the node IP when ipWithNat is empty. + if len(ipWithNat) > 0 { + publishContext[identifiers.KeyHostIP] = ipWithNat[0] + } else { + publishContext[identifiers.KeyHostIP] = ip + } + } if n.ExternalAccess != "" { parsedExternalAccess, _ := identifiers.GetIPListWithMaskFromString(n.ExternalAccess) publishContext[identifiers.KeyNatIP] = parsedExternalAccess diff --git a/pkg/controller/replication.go b/pkg/controller/replication.go index 8faf5bda..36fa3aa6 100644 --- a/pkg/controller/replication.go +++ b/pkg/controller/replication.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2024 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,10 +22,8 @@ import ( "strings" "github.com/dell/csi-powerstore/v2/pkg/array" - "github.com/dell/csm-sharednfs/nfs" csiext "github.com/dell/dell-csi-extensions/replication" "github.com/dell/gopowerstore" - log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -34,31 +32,24 @@ import ( func (s *Service) CreateRemoteVolume(ctx context.Context, req *csiext.CreateRemoteVolumeRequest, ) (*csiext.CreateRemoteVolumeResponse, error) { + log := log.WithContext(ctx) volID := req.GetVolumeHandle() + log.Infof("CreateRemoteVolume: Full Volumerequest: %+v", req) if volID == "" { return nil, status.Error(codes.InvalidArgument, "volume ID is required") } - params := req.GetParameters() volumeHandle, err := array.ParseVolumeID(ctx, volID, s.DefaultArray(), nil) if err != nil { - log.Error(err) + log.Error(err.Error()) return nil, err } id := volumeHandle.LocalUUID arrayID := volumeHandle.LocalArrayGlobalID - protocol := volumeHandle.Protocol + protocol := strings.ToLower(volumeHandle.Protocol) + log.Infof("CreateRemoteVolume: Parsed volume ID: LocalUUID=%s, ArrayID=%s, Protocol=%s", id, arrayID, protocol) volPrefix := "" - if accessMode, ok := params[nfs.CsiNfsParameter]; ok && accessMode != "" { - // host-based nfs volumes should have the "shared-nfs" parameter - // and an "nfs-" prefix in the volume ID that we need to remove - // for gopowerstore queries to succeed. - // Remove the prefix here and restore it when building the volume ID - // for the function response. - volPrefix = array.GetVolumeUUIDPrefix(id) - id = strings.TrimPrefix(id, volPrefix) - } arr, ok := s.Arrays()[arrayID] if !ok { @@ -66,40 +57,106 @@ func (s *Service) CreateRemoteVolume(ctx context.Context, return nil, status.Error(codes.InvalidArgument, "failed to find array with given IP") } - vgs, err := arr.GetClient().GetVolumeGroupsByVolumeID(ctx, id) - if err != nil { - return nil, err - } - if len(vgs.VolumeGroup) == 0 { - return nil, status.Error(codes.Unimplemented, "replication of volumes that aren't assigned to group is not implemented yet") - } - vg := vgs.VolumeGroup[0] + var remoteVolumeID string + var volSize int64 + var remoteSystemID string - rs, err := arr.Client.GetReplicationSessionByLocalResourceID(ctx, vg.ID) - if err != nil { - return nil, err - } + if protocol == "nfs" { + log.Infof("CreateRemoteVolume: Checking NFS export for file system ID: %s", id) + export, err := arr.Client.GetNFSExportByFileSystemID(ctx, id) + if err != nil { + log.Warnf("CreateRemoteVolume: NFS export not found, attempting to create one for file system ID: %s", id) - var remoteVolumeID string - for _, sp := range rs.StorageElementPairs { - if sp.LocalStorageElementID == id { - remoteVolumeID = sp.RemoteStorageElementID + fs, err := arr.Client.GetFS(ctx, id) + if err != nil { + log.Errorf("Failed to get file system by ID %s: %v", id, err) + return nil, status.Errorf(codes.NotFound, "file system not found") + } + prefix := "auto_export_" + var exportName string + if len(id) >= 8 { + exportName = prefix + id[:8] + } else { + exportName = prefix + id + } + + createReq := gopowerstore.NFSExportCreate{ + FileSystemID: id, + Name: exportName, + Path: "/" + fs.Name, + DefaultAccess: gopowerstore.NFSExportDefaultAccessEnum("No_Access"), + MinSecurity: "Sys", + AnonymousUID: -2, + AnonymousGID: -2, + IsNoSUID: false, + NFSOwnerUsername: "root", + } + exportID, createErr := arr.Client.CreateNFSExport(ctx, &createReq) + if createErr != nil { + log.Errorf("CreateRemoteVolume: Failed to create NFS export: %v", createErr) + return nil, status.Errorf(codes.Internal, "failed to create NFS export for file system ID %s", id) + } + + log.Infof("CreateRemoteVolume: Successfully created NFS export with ID: %s", exportID) + + // Retrieve the newly created export + export, err = arr.Client.GetNFSExportByFileSystemID(ctx, id) + if err != nil { + log.Errorf("CreateRemoteVolume: Failed to retrieve newly created NFS export: %v", err) + return nil, status.Errorf(codes.Internal, "unable to retrieve NFS export after creation for file system ID %s", id) + } + } else { + log.Infof("CreateRemoteVolume: Retrieved existing export: %+v", export) } - } - if remoteVolumeID == "" { - return nil, status.Errorf(codes.Internal, "couldn't find volume id %s in storage element pairs of replication session", id) - } + rs, err := arr.Client.GetReplicationSessionByLocalResourceID(ctx, id) + if err != nil { + log.Errorf("CreateRemoteVolume: No replication session found for file system ID: %s, error: %v", id, err) + return nil, status.Error(codes.Internal, "no replication session found for file system") + } - vol, err := arr.Client.GetVolume(ctx, id) - if err != nil { - return nil, status.Errorf(codes.Internal, "can't query volume: %s", err.Error()) + remoteSystemID = rs.RemoteSystemID + remoteVolumeID = id + volSize = 0 + } else { + // Block volume logic + vgs, err := arr.GetClient().GetVolumeGroupsByVolumeID(ctx, id) + if err != nil { + return nil, err + } + if len(vgs.VolumeGroup) == 0 { + return nil, status.Error(codes.Unimplemented, "replication of volumes that aren't assigned to group is not implemented yet") + } + vg := vgs.VolumeGroup[0] + + rs, err := arr.Client.GetReplicationSessionByLocalResourceID(ctx, vg.ID) + if err != nil { + return nil, err + } + remoteSystemID = rs.RemoteSystemID + + for _, sp := range rs.StorageElementPairs { + if sp.LocalStorageElementID == id { + remoteVolumeID = sp.RemoteStorageElementID + } + } + if remoteVolumeID == "" { + return nil, status.Errorf(codes.Internal, "couldn't find volume id %s in storage element pairs of replication session", id) + } + + vol, err := arr.Client.GetVolume(ctx, id) + if err != nil { + return nil, status.Errorf(codes.Internal, "can't query volume: %s", err.Error()) + } + volSize = vol.Size } + + // Get local and remote system info localSystem, err := arr.Client.GetCluster(ctx) if err != nil { return nil, err } - remoteSystem, err := arr.Client.GetRemoteSystem(ctx, rs.RemoteSystemID) + remoteSystem, err := arr.Client.GetRemoteSystem(ctx, remoteSystemID) if err != nil { return nil, err } @@ -109,11 +166,13 @@ func (s *Service) CreateRemoteVolume(ctx context.Context, s.replicationContextPrefix + "arrayID": remoteSystem.SerialNumber, s.replicationContextPrefix + "managementAddress": remoteSystem.ManagementAddress, } + remoteVolume := getRemoteCSIVolume( volPrefix+remoteVolumeID+"/"+remoteParams[s.replicationContextPrefix+"arrayID"]+"/"+protocol, - vol.Size, + volSize, ) remoteVolume.VolumeContext = remoteParams + return &csiext.CreateRemoteVolumeResponse{ RemoteVolume: remoteVolume, }, nil @@ -123,29 +182,21 @@ func (s *Service) CreateRemoteVolume(ctx context.Context, func (s *Service) CreateStorageProtectionGroup(ctx context.Context, req *csiext.CreateStorageProtectionGroupRequest, ) (*csiext.CreateStorageProtectionGroupResponse, error) { + log := log.WithContext(ctx) volID := req.GetVolumeHandle() if volID == "" { return nil, status.Error(codes.InvalidArgument, "volume ID is required") } - params := req.GetParameters() volumeHandle, err := array.ParseVolumeID(ctx, volID, s.DefaultArray(), nil) if err != nil { - log.Error(err) + log.Error(err.Error()) return nil, err } id := volumeHandle.LocalUUID arrayID := volumeHandle.LocalArrayGlobalID - protocol := volumeHandle.Protocol - - if accessMode, ok := params[nfs.CsiNfsParameter]; ok && accessMode != "" { - // host-based nfs volumes should have the "shared-nfs" parameter - // and a "nfs-" prefix in the volume ID that we need to remove - // for gopowerstore queries to succeed - volPrefix := array.GetVolumeUUIDPrefix(id) - id = strings.TrimPrefix(id, volPrefix) - } + protocol := strings.ToLower(volumeHandle.Protocol) arr, ok := s.Arrays()[arrayID] if !ok { @@ -153,54 +204,100 @@ func (s *Service) CreateStorageProtectionGroup(ctx context.Context, return nil, status.Error(codes.InvalidArgument, "failed to find array with given ID") } - if protocol == "nfs" { - return nil, status.Error(codes.InvalidArgument, "replication is not supported for NFS volumes") - } + var localGroupID, remoteGroupID string + var localParams, remoteParams map[string]string - vgs, err := arr.GetClient().GetVolumeGroupsByVolumeID(ctx, id) + localSystem, err := arr.Client.GetCluster(ctx) if err != nil { return nil, err } - if len(vgs.VolumeGroup) == 0 { - return nil, status.Error(codes.Unimplemented, "replication of volumes that aren't assigned to group is not implemented yet") - } - vg := vgs.VolumeGroup[0] - rs, err := arr.Client.GetReplicationSessionByLocalResourceID(ctx, vg.ID) - if err != nil { - return nil, err - } + if protocol == "nfs" { + log.Infof("CreateRemoteVolume: Checking NFS export for file system ID: %s", id) + export, err := arr.Client.GetNFSExportByFileSystemID(ctx, id) + if err != nil { + log.Errorf("CreateRemoteVolume: Error retrieving NFS export: %v", err) + } else { + log.Infof("CreateRemoteVolume: Retrieved export: %+v", export) + } + nasServerID := export.ID - localSystem, err := arr.Client.GetCluster(ctx) - if err != nil { - return nil, err - } + rs, err := arr.Client.GetReplicationSessionByLocalResourceID(ctx, id) + if err != nil { + return nil, status.Error(codes.Internal, "no replication session found for NAS server") + } - remoteSystem, err := arr.Client.GetRemoteSystem(ctx, rs.RemoteSystemID) - if err != nil { - return nil, err - } - localParams := map[string]string{ - s.replicationContextPrefix + "systemName": localSystem.Name, - s.replicationContextPrefix + "managementAddress": localSystem.ManagementAddress, - s.replicationContextPrefix + "remoteSystemName": remoteSystem.Name, - s.replicationContextPrefix + "remoteManagementAddress": remoteSystem.ManagementAddress, - s.replicationContextPrefix + "globalID": arrayID, - s.replicationContextPrefix + "remoteGlobalID": remoteSystem.SerialNumber, - s.replicationContextPrefix + "VolumeGroupName": vg.Name, - } - remoteParams := map[string]string{ - s.replicationContextPrefix + "systemName": remoteSystem.Name, - s.replicationContextPrefix + "managementAddress": remoteSystem.ManagementAddress, - s.replicationContextPrefix + "remoteSystemName": localSystem.Name, - s.replicationContextPrefix + "remoteManagementAddress": localSystem.ManagementAddress, - s.replicationContextPrefix + "globalID": remoteSystem.SerialNumber, - s.replicationContextPrefix + "VolumeGroupName": vg.Name, + remoteSystem, err := arr.Client.GetRemoteSystem(ctx, rs.RemoteSystemID) + if err != nil { + return nil, err + } + + localGroupID = rs.LocalResourceID + remoteGroupID = rs.RemoteResourceID + + localParams = map[string]string{ + s.replicationContextPrefix + "systemName": localSystem.Name, + s.replicationContextPrefix + "managementAddress": localSystem.ManagementAddress, + s.replicationContextPrefix + "remoteSystemName": remoteSystem.Name, + s.replicationContextPrefix + "remoteManagementAddress": remoteSystem.ManagementAddress, + s.replicationContextPrefix + "globalID": arrayID, + s.replicationContextPrefix + "remoteGlobalID": remoteSystem.SerialNumber, + s.replicationContextPrefix + "NasServerID": nasServerID, + } + remoteParams = map[string]string{ + s.replicationContextPrefix + "systemName": remoteSystem.Name, + s.replicationContextPrefix + "managementAddress": remoteSystem.ManagementAddress, + s.replicationContextPrefix + "remoteSystemName": localSystem.Name, + s.replicationContextPrefix + "remoteManagementAddress": localSystem.ManagementAddress, + s.replicationContextPrefix + "globalID": remoteSystem.SerialNumber, + s.replicationContextPrefix + "NasServerID": rs.RemoteResourceID, + } + } else { + // Block volume logic + vgs, err := arr.GetClient().GetVolumeGroupsByVolumeID(ctx, id) + if err != nil { + return nil, err + } + if len(vgs.VolumeGroup) == 0 { + return nil, status.Error(codes.Unimplemented, "replication of volumes that aren't assigned to group is not implemented yet") + } + vg := vgs.VolumeGroup[0] + + rs, err := arr.Client.GetReplicationSessionByLocalResourceID(ctx, vg.ID) + if err != nil { + return nil, err + } + + remoteSystem, err := arr.Client.GetRemoteSystem(ctx, rs.RemoteSystemID) + if err != nil { + return nil, err + } + + localGroupID = rs.LocalResourceID + remoteGroupID = rs.RemoteResourceID + + localParams = map[string]string{ + s.replicationContextPrefix + "systemName": localSystem.Name, + s.replicationContextPrefix + "managementAddress": localSystem.ManagementAddress, + s.replicationContextPrefix + "remoteSystemName": remoteSystem.Name, + s.replicationContextPrefix + "remoteManagementAddress": remoteSystem.ManagementAddress, + s.replicationContextPrefix + "globalID": arrayID, + s.replicationContextPrefix + "remoteGlobalID": remoteSystem.SerialNumber, + s.replicationContextPrefix + "VolumeGroupName": vg.Name, + } + remoteParams = map[string]string{ + s.replicationContextPrefix + "systemName": remoteSystem.Name, + s.replicationContextPrefix + "managementAddress": remoteSystem.ManagementAddress, + s.replicationContextPrefix + "remoteSystemName": localSystem.Name, + s.replicationContextPrefix + "remoteManagementAddress": localSystem.ManagementAddress, + s.replicationContextPrefix + "globalID": remoteSystem.SerialNumber, + s.replicationContextPrefix + "VolumeGroupName": vg.Name, + } } return &csiext.CreateStorageProtectionGroupResponse{ - LocalProtectionGroupId: rs.LocalResourceID, - RemoteProtectionGroupId: rs.RemoteResourceID, + LocalProtectionGroupId: localGroupID, + RemoteProtectionGroupId: remoteGroupID, LocalProtectionGroupAttributes: localParams, RemoteProtectionGroupAttributes: remoteParams, }, nil @@ -341,6 +438,7 @@ func (s *Service) GetReplicationCapabilities(_ context.Context, _ *csiext.GetRep func (s *Service) ExecuteAction(ctx context.Context, req *csiext.ExecuteActionRequest, ) (*csiext.ExecuteActionResponse, error) { + log := log.WithContext(ctx) var reqID string localParams := req.GetProtectionGroupAttributes() protectionGroupID := req.GetProtectionGroupId() @@ -468,14 +566,13 @@ func validateRSState(session *gopowerstore.ReplicationSession, action gopowersto return false, true, nil } -// DeleteStorageProtectionGroup deletes storage protection group -func (s *Service) DeleteStorageProtectionGroup(ctx context.Context, +func (s *Service) DeleteStorageProtectionGroup( + ctx context.Context, req *csiext.DeleteStorageProtectionGroupRequest, ) (*csiext.DeleteStorageProtectionGroupResponse, error) { localParams := req.GetProtectionGroupAttributes() groupID := req.GetProtectionGroupId() globalID, ok := localParams[s.replicationContextPrefix+"globalID"] - if !ok { return nil, status.Error(codes.InvalidArgument, "missing globalID in protection group attributes") } @@ -488,11 +585,24 @@ func (s *Service) DeleteStorageProtectionGroup(ctx context.Context, "GlobalID": globalID, "ProtectedStorageGroup": groupID, } + log := log.WithContext(ctx).WithFields(fields) + + log.Info("Deleting storage protection group") - log.WithFields(fields).Info("Deleting storage protection group") + nasServerID, hasNas := localParams[s.replicationContextPrefix+"NasServerID"] + // Skip deletion logic for NFS replication sessions + // Storage Protection Group (i.e., Protection Policy) cannot be deleted from the PowerStore array + // if it is assigned to a NAS server. Modifying NAS to unassign the policy is currently unsupported, + // so deletion is effectively blocked for sync/async NAS contexts. + if hasNas && nasServerID != "" { + log.Info("NFS context detected — skipping deletion logic") + return &csiext.DeleteStorageProtectionGroupResponse{}, nil + } + // Block: Unassign PP and delete VolumeGroup vg, err := arr.GetClient().GetVolumeGroup(ctx, groupID) if apiErr, ok := err.(gopowerstore.APIError); ok && !apiErr.NotFound() { + log.Errorf("Failed to get Volume Group: %v", apiErr) return nil, status.Errorf(codes.Internal, "Error: Unable to get Volume Group") } if vg.ID != "" { @@ -501,41 +611,50 @@ func (s *Service) DeleteStorageProtectionGroup(ctx context.Context, ProtectionPolicyID: "", }, groupID) if apiErr, ok := err.(gopowerstore.APIError); ok && !apiErr.NotFound() { + log.Errorf("Unable to un-assign PP from Volume Group: %v", apiErr) return nil, status.Errorf(codes.Internal, "Error: Unable to un-assign PP from Volume Group") } } _, err = arr.Client.DeleteVolumeGroup(ctx, groupID) - if apiError, ok := err.(gopowerstore.APIError); ok && !apiError.NotFound() { - return nil, status.Errorf(codes.Internal, "Error: %s: Unable to delete Volume Group", apiError.Error()) + if apiErr, ok := err.(gopowerstore.APIError); ok && !apiErr.NotFound() { + log.Errorf("Unable to delete Volume Group: %v", apiErr) + return nil, status.Errorf(codes.Internal, "Error: Unable to delete Volume Group") } } - log.WithFields(fields).Info("Deleting protection policy") - vgName, ok := localParams[s.replicationContextPrefix+"VolumeGroupName"] if !ok { return nil, status.Errorf(codes.Internal, "Error: Unable to get volume group name") } + + // Delete Protection Policy + log.Info("Deleting protection policy") pp, err := arr.GetClient().GetProtectionPolicyByName(ctx, "pp-"+vgName) if apiErr, ok := err.(gopowerstore.APIError); ok && !apiErr.NotFound() { - return nil, status.Errorf(codes.Internal, "Error: Unable to get the PP") + log.Errorf("Error retrieving protection policy: %v", apiErr) + return nil, status.Errorf(codes.Internal, "Error: Unable to get protection policy") } - if pp.ID != "" && len(pp.Volumes) == 0 && len(pp.VolumeGroups) == 0 { + if pp.ID != "" && + len(pp.Volumes) == 0 && + len(pp.VolumeGroups) == 0 { _, err := arr.Client.DeleteProtectionPolicy(ctx, pp.ID) if apiErr, ok := err.(gopowerstore.APIError); ok && !apiErr.NotFound() { - return nil, status.Errorf(codes.Internal, "Error: Unable to delete PP") + log.Errorf("Unable to delete protection policy: %v", apiErr) + return nil, status.Errorf(codes.Internal, "Error: Unable to delete protection policy") } } - log.WithFields(fields).Info("Deleting replication rule") - + // Delete Replication Rule + log.Info("Deleting replication rule") rr, err := arr.GetClient().GetReplicationRuleByName(ctx, "rr-"+vgName) if apiErr, ok := err.(gopowerstore.APIError); ok && !apiErr.NotFound() { - return nil, status.Errorf(codes.Internal, "Error: RR not found") + log.Errorf("Error retrieving replication rule: %v", apiErr) + return nil, status.Errorf(codes.Internal, "Error: Unable to get replication rule") } if rr.ID != "" && len(rr.ProtectionPolicies) == 0 { _, err = arr.GetClient().DeleteReplicationRule(ctx, rr.ID) if apiErr, ok := err.(gopowerstore.APIError); ok && !apiErr.NotFound() { + log.Errorf("Unable to delete replication rule: %v", apiErr) return nil, status.Errorf(codes.Internal, "Error: Unable to delete replication rule") } } @@ -547,6 +666,7 @@ func (s *Service) DeleteStorageProtectionGroup(ctx context.Context, func (s *Service) DeleteLocalVolume(ctx context.Context, req *csiext.DeleteLocalVolumeRequest, ) (*csiext.DeleteLocalVolumeResponse, error) { + log := log.WithContext(ctx) log.Info("Deleting local volume " + req.VolumeHandle + " per request from remote replication controller") // req.VolumeHandle is of format //. We only need the IDs. @@ -608,6 +728,7 @@ func (s *Service) DeleteLocalVolume(ctx context.Context, func (s *Service) GetStorageProtectionGroupStatus(ctx context.Context, req *csiext.GetStorageProtectionGroupStatusRequest, ) (*csiext.GetStorageProtectionGroupStatusResponse, error) { + log := log.WithContext(ctx) localParams := req.GetProtectionGroupAttributes() groupID := req.GetProtectionGroupId() diff --git a/pkg/controller/replication_test.go b/pkg/controller/replication_test.go index 4ecf0cb4..f9ff609a 100755 --- a/pkg/controller/replication_test.go +++ b/pkg/controller/replication_test.go @@ -19,17 +19,11 @@ package controller import ( "context" "net/http" - "reflect" - "testing" "github.com/dell/csi-powerstore/v2/pkg/array" - "github.com/dell/csi-powerstore/v2/pkg/identifiers" - "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" - "github.com/dell/csm-sharednfs/nfs" csiext "github.com/dell/dell-csi-extensions/replication" "github.com/dell/gopowerstore" "github.com/dell/gopowerstore/api" - gopowerstoreMock "github.com/dell/gopowerstore/mocks" ginkgo "github.com/onsi/ginkgo" gomega "github.com/onsi/gomega" "github.com/stretchr/testify/mock" @@ -669,7 +663,7 @@ var _ = ginkgo.Describe("Replication", func() { gomega.Expect(res).To(gomega.BeNil()) gomega.Expect(err).ToNot(gomega.BeNil()) gomega.Expect(err.Error()).To( - gomega.ContainSubstring("Error: : Unable to delete Volume Group")) + gomega.ContainSubstring("Error: Unable to delete Volume Group")) }) }) ginkgo.When("Can't get the protection policy", func() { @@ -704,7 +698,7 @@ var _ = ginkgo.Describe("Replication", func() { gomega.Expect(res).To(gomega.BeNil()) gomega.Expect(err).ToNot(gomega.BeNil()) gomega.Expect(err.Error()).To( - gomega.ContainSubstring("Error: Unable to get the PP")) + gomega.ContainSubstring("Error: Unable to get protection policy")) }) }) @@ -742,7 +736,7 @@ var _ = ginkgo.Describe("Replication", func() { gomega.Expect(res).To(gomega.BeNil()) gomega.Expect(err).ToNot(gomega.BeNil()) gomega.Expect(err.Error()).To( - gomega.ContainSubstring("Error: RR not found")) + gomega.ContainSubstring("Error: Unable to get replication rule")) }) }) ginkgo.When("The replication rule can't be deleted", func() { @@ -786,6 +780,18 @@ var _ = ginkgo.Describe("Replication", func() { gomega.ContainSubstring("Error: Unable to delete replication rule")) }) }) + ginkgo.When("NFS context is detected via NasServerID", func() { + ginkgo.It("should skip deletion logic and return success", func() { + req := new(csiext.DeleteStorageProtectionGroupRequest) + params := make(map[string]string) + params["globalID"] = firstValidID + params["NasServerID"] = "nas-server-id" + req.ProtectionGroupAttributes = params + res, err := ctrlSvc.DeleteStorageProtectionGroup(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).ToNot(gomega.BeNil()) + }) + }) }) ginkgo.Describe("calling GetReplicationCapabilities()", func() { ginkgo.When("basic parameters are declared", func() { @@ -1132,164 +1138,3 @@ var _ = ginkgo.Describe("Replication", func() { }) }) }) - -func TestService_CreateRemoteVolume(t *testing.T) { - const GiB int64 = 1073741824 - - replicationContextPrefix := "powerstore/" - - localVolUUID := "aaaaaaaa-0000-bbbb-1111-cccccccccccc" - remoteVolUUID := "00000000-aaaa-1111-bbbb-222222222222" - - powerstoreLocalSystemID := "PS000000000001" - powerstoreRemoteSystemID := "PS000000000002" - - localVolumeID := nfs.CsiNfsPrefixDash + localVolUUID + "/" + powerstoreLocalSystemID + "/scsi" - remoteVolumeID := nfs.CsiNfsPrefixDash + remoteVolUUID + "/" + powerstoreRemoteSystemID + "/scsi" - - powerstoreLocalSystemName := "local-system" - - powerstoreLocalEndpoint := "127.0.0.1" - powerstoreRemoteEndpoint := "127.0.0.2" - - powerstoreDefaultGID := powerstoreLocalSystemID - - sourceArray := array.PowerStoreArray{ - Endpoint: powerstoreLocalEndpoint, - GlobalID: powerstoreDefaultGID, - Username: "user", - Password: "password", - BlockProtocol: identifiers.ISCSITransport, - Insecure: true, - IsDefault: true, - Client: nil, - } - - type fields struct { - Fs fs.Interface - externalAccess string - nfsAcls string - replicationContextPrefix string - replicationPrefix string - isHealthMonitorEnabled bool - isAutoRoundOffFsSizeEnabled bool - } - type args struct { - ctx context.Context - req *csiext.CreateRemoteVolumeRequest - } - type testcase struct { - name string - fields fields - args args - before func(s *Service) - want *csiext.CreateRemoteVolumeResponse - wantErr bool - } - tests := []testcase{ - { - name: "creates a remote host-based nfs volume", - fields: fields{ - replicationContextPrefix: replicationContextPrefix, - }, - args: args{ - context.Background(), - &csiext.CreateRemoteVolumeRequest{ - VolumeHandle: localVolumeID, - Parameters: map[string]string{ - nfs.CsiNfsParameter: "RWX", - }, - }, - }, - before: func(s *Service) { - // need to initialize the arrays and default array here, - // because we cannot copy the mutexs in array.Locker, - // and members of array.Locker are unexported, so we must - // use the provided setter funcs. - - // make a copy so as not to modify the stub - defaultArray := sourceArray - - // setup mock responses to gopowerstore queries - mockGopowerstoreClient := gopowerstoreMock.NewClient(t) - mockGopowerstoreClient.On("GetVolumeGroupsByVolumeID", mock.Anything, localVolUUID).Return(gopowerstore.VolumeGroups{ - VolumeGroup: []gopowerstore.VolumeGroup{{ID: "vg-uuid"}}, - }, nil) - mockGopowerstoreClient.On("GetReplicationSessionByLocalResourceID", mock.Anything, "vg-uuid").Return( - gopowerstore.ReplicationSession{ - // return a replication session linking the volumes on local and remote systems - StorageElementPairs: []gopowerstore.StorageElementPair{ - { - LocalStorageElementID: localVolUUID, - RemoteStorageElementID: remoteVolUUID, - }, - }, - RemoteSystemID: powerstoreRemoteSystemID, - }, - nil, - ) - mockGopowerstoreClient.On("GetVolume", mock.Anything, localVolUUID).Return( - gopowerstore.Volume{ - // only the size is required for this test. extra info is omitted. - Size: 5 * GiB, - }, nil, - ) - mockGopowerstoreClient.On("GetCluster", mock.Anything).Return(gopowerstore.Cluster{Name: powerstoreLocalSystemName}, nil) - mockGopowerstoreClient.On("GetRemoteSystem", mock.Anything, powerstoreRemoteSystemID).Return( - gopowerstore.RemoteSystem{ - SerialNumber: powerstoreRemoteSystemID, - ManagementAddress: powerstoreRemoteEndpoint, - }, - nil, - ) - - // assign the ephemeral mock client to the default array - defaultArray.Client = mockGopowerstoreClient - // add the default array to the list of arrays - arrays := map[string]*array.PowerStoreArray{ - powerstoreDefaultGID: &defaultArray, - } - - // initialize the Locker/arrays - s.Locker.SetArrays(arrays) - s.Locker.SetDefaultArray(&defaultArray) - }, - want: &csiext.CreateRemoteVolumeResponse{ - RemoteVolume: &csiext.Volume{ - CapacityBytes: 5 * GiB, - VolumeId: remoteVolumeID, - VolumeContext: map[string]string{ - "remoteSystem": powerstoreLocalSystemName, - replicationContextPrefix + "arrayID": powerstoreRemoteSystemID, - replicationContextPrefix + "managementAddress": powerstoreRemoteEndpoint, - }, - }, - }, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := &Service{ - Fs: tt.fields.Fs, - externalAccess: tt.fields.externalAccess, - nfsAcls: tt.fields.nfsAcls, - Locker: *new(array.Locker), - replicationContextPrefix: tt.fields.replicationContextPrefix, - replicationPrefix: tt.fields.replicationPrefix, - isHealthMonitorEnabled: tt.fields.isHealthMonitorEnabled, - isAutoRoundOffFsSizeEnabled: tt.fields.isAutoRoundOffFsSizeEnabled, - } - tt.before(s) - - got, err := s.CreateRemoteVolume(tt.args.ctx, tt.args.req) - if (err != nil) != tt.wantErr { - t.Errorf("Service.CreateRemoteVolume() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("Service.CreateRemoteVolume() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/helpers/utils.go b/pkg/helpers/utils.go index 4480b5ab..d9f63802 100644 --- a/pkg/helpers/utils.go +++ b/pkg/helpers/utils.go @@ -21,8 +21,13 @@ package helpers import ( "errors" "net" + + "github.com/dell/csmlog" ) +// Instantiate csmlog at package level +var log = csmlog.GetLogger() + // InterfaceProvider allows mocking net.Interfaces type InterfaceProvider interface { Interfaces() ([]net.Interface, error) diff --git a/pkg/helpers/utils_test.go b/pkg/helpers/utils_test.go index 9a2ee80e..ff7e5220 100644 --- a/pkg/helpers/utils_test.go +++ b/pkg/helpers/utils_test.go @@ -183,10 +183,9 @@ func TestDefaultProvider_Addrs(t *testing.T) { assert.NotEmpty(t, interfaces) for _, iface := range interfaces { - addrs, err := provider.Addrs(iface) + _, err := provider.Addrs(iface) // Some interfaces may not have addresses, but the call shouldn't fail assert.NoError(t, err) - assert.NotNil(t, addrs) } } diff --git a/pkg/identifiers/envvars.go b/pkg/identifiers/envvars.go index 82630d88..3063cab6 100644 --- a/pkg/identifiers/envvars.go +++ b/pkg/identifiers/envvars.go @@ -68,6 +68,10 @@ const ( // Used to provide NFS volumes behind NAT EnvExternalAccess = "X_CSI_POWERSTORE_EXTERNAL_ACCESS" // #nosec G101 + // EnvExclusiveAccess indicates whether only externalAccess entries should be added to the NFS export. + // If true, node IP is excluded, and only IP/CIDR from externalAccess is used. + EnvExclusiveAccess = "X_CSI_POWERSTORE_EXCLUSIVE_ACCESS" + // EnvArrayConfigFilePath is filepath to powerstore arrays config file EnvArrayConfigFilePath = "X_CSI_POWERSTORE_CONFIG_PATH" @@ -113,9 +117,6 @@ const ( // EnvMultiNASCooldownPeriod specifies the cooldown period for multiple NAS devices. EnvMultiNASCooldownPeriod = "X_CSI_MULTI_NAS_COOLDOWN_PERIOD" - // EnvNFSExportDirectory is the path to the folder where the nfs volumes are mounted - EnvNFSExportDirectory = "X_CSI_NFS_EXPORT_DIRECTORY" - // EnvDriverNamespace is the namespace where the powerstore driver is deployed EnvDriverNamespace = "X_CSI_DRIVER_NAMESPACE" @@ -124,4 +125,16 @@ const ( // EnvPodmonArrayConnectivityTimeout specifies the timeout for array connectivity for podmon EnvPodmonArrayConnectivityTimeout = "X_CSI_PODMON_ARRAY_CONNECTIVITY_TIMEOUT" + + // EnvVolumeDisconnectMaxRetries specifies the maximum number of retry attempts for volume disconnection + EnvVolumeDisconnectMaxRetries = "X_CSI_VOLUME_DISCONNECT_MAX_RETRIES" + + // EnvVolumeDisconnectRetryInterval specifies the wait time (in seconds) between volume disconnection retries + EnvVolumeDisconnectRetryInterval = "X_CSI_VOLUME_DISCONNECT_RETRY_INTERVAL" + + // EnvVolumeDisconnectTimeoutSeconds specifies the timeout duration (in seconds) for each volume disconnection attempt + EnvVolumeDisconnectTimeoutSeconds = "X_CSI_VOLUME_DISCONNECT_TIMEOUT_SECONDS" + + // EnvCSMDREnabled indicates if CSM-DR is enabled + EnvCSMDREnabled = "X_CSM_DR_ENABLED" ) diff --git a/pkg/identifiers/fs/fs.go b/pkg/identifiers/fs/fs.go index 4f68626c..9206d165 100644 --- a/pkg/identifiers/fs/fs.go +++ b/pkg/identifiers/fs/fs.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,10 +32,13 @@ import ( "syscall" "time" + "github.com/dell/csmlog" "github.com/dell/gofsutil" - log "github.com/sirupsen/logrus" ) +// Instantiate csmlog on a package level +var log = csmlog.GetLogger() + // A FileInfo describes a file and is returned by Stat and Lstat. type FileInfo interface { Name() string // base name of the file @@ -214,13 +217,18 @@ func (fs *Fs) MkFileIdempotent(path string) (bool, error) { if fs.IsNotExist(err) { file, err := fs.OpenFile(path, os.O_CREATE, 0o600) if err != nil { - log.WithField("path", path).WithError(err).Error("Unable to create file") + log.WithFields(csmlog.Fields{ + "path": path, + }).Error("Unable to create file" + err.Error()) return false, err } if err = file.Close(); err != nil { return false, fmt.Errorf("could not close file") } - log.WithField("path", path).Debug("created file") + log.WithFields(csmlog.Fields{ + "path": path, + }).Debug("created file") + return true, nil } if st.IsDir() { diff --git a/pkg/identifiers/identifiers.go b/pkg/identifiers/identifiers.go index c750873f..19818360 100644 --- a/pkg/identifiers/identifiers.go +++ b/pkg/identifiers/identifiers.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,34 +28,39 @@ import ( "net" "os" "regexp" + "slices" "sort" "strconv" "strings" "sync" "time" - "github.com/apparentlymart/go-cidr/cidr" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/core" "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + "github.com/dell/csmlog" "github.com/dell/gobrick" csictx "github.com/dell/gocsi/context" csiutils "github.com/dell/gocsi/utils/csi" "github.com/dell/gopowerstore" - log "github.com/sirupsen/logrus" + "github.com/apparentlymart/go-cidr/cidr" + "github.com/container-storage-interface/spec/lib/go/csi" ) +// Instantiate csmlog on a package level +var log = csmlog.GetLogger() + // Name contains default name of the driver, can be overridden var Name = "csi-powerstore.dellemc.com" // APIPort port for API calls var APIPort string +// Update when the manifest version changes. +var ManifestSemver string + // Manifest contains additional information about the driver var Manifest = map[string]string{ - "url": "https://github.com/dell/csi-powerstore", - "semver": core.SemVer, - "commit": core.CommitSha32, + "semver": ManifestSemver, "formed": core.CommitTime.Format(time.RFC1123), } @@ -197,6 +202,9 @@ const ( // ArrayStatus is the endPoint for polling to check array status ArrayStatus = "/array-status" + + // KeyNodeID represents key for node id + KeyNodeID = "csi.volume.kubernetes.io/nodeid" ) // PodmonArrayConnectivityTimeout specifies timeout for making http requests to node services by podmon @@ -211,6 +219,15 @@ var PowerstoreRESTApiTimeout = GetPowerStoreRESTApiTimeout() // DefaultPowerstoreRESTApiTimeout specifies default timeout for making http requests by Powerstore client var DefaultPowerstoreRESTApiTimeout = 120 * time.Second +// DefaultVolumeDisconnectMaxRetries specifies the default maximum number of retry attempts for volume disconnection +var DefaultVolumeDisconnectMaxRetries = 5 + +// DefaultVolumeDisconnectRetryInterval specifies the default wait time between volume disconnection retries +var DefaultVolumeDisconnectRetryInterval = 5 * time.Second + +// DefaultVolumeDisconnectTimeout specifies the default timeout duration for each volume disconnection attempt +var DefaultVolumeDisconnectTimeout = 120 * time.Second + // TransportType differentiates different SCSI transport protocols (FC, iSCSI, Auto, None) type TransportType string @@ -241,8 +258,37 @@ func RmSockFile(f fs.Interface) { // GetIPListFromString returns list of ips in string form found in input string // A return value of nil indicates no match func GetIPListFromString(input string) []string { - re := regexp.MustCompile(`\b((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|localhost|([a-zA-Z0-9-]+\.)+[a-zA-Z]{2,})\b`) - return re.FindAllString(input, -1) + ipRe := regexp.MustCompile(`\b((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|localhost\b)`) + urlRe := regexp.MustCompile(`https?://([a-zA-Z0-9.-]+)(:[0-9]+)?(/.*)?`) + + ipMatches := ipRe.FindAllString(input, -1) + urlMatches := urlRe.FindAllStringSubmatch(input, -1) + + var matches []string + matches = append(matches, ipMatches...) + for _, match := range urlMatches { + if len(match) > 1 { + if strings.Contains(match[1], ".") { + if !isDomain(match[1]) { + continue + } + } + if !slices.Contains(matches, match[1]) { + matches = append(matches, match[1]) + } + } + } + + return matches +} + +func isDomain(str string) bool { + for _, r := range str { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') { + return true + } + } + return false } func parseMask(ipaddr string) (mask string, err error) { @@ -296,43 +342,18 @@ func GetIPListWithMaskFromString(input string) (string, error) { return ip, nil } -// SetLogFields returns modified context with fields inserted as values by using contextLogFieldsKey key -func SetLogFields(ctx context.Context, fields log.Fields) context.Context { - if ctx == nil { - ctx = context.Background() - } - return context.WithValue(ctx, contextLogFieldsKey, fields) -} - // RandomString returns a random string of specified length. // String is generated by using crypto/rand. func RandomString(length int) string { b := make([]byte, length) _, err := rand.Read(b) if err != nil { - log.Errorf("Can't generate random string; error = %v", err) + return err.Error() } suff := fmt.Sprintf("%x", b[0:]) return suff } -// GetLogFields extracts log fields from context by using contextLogFieldsKey key -func GetLogFields(ctx context.Context) log.Fields { - if ctx == nil { - return log.Fields{} - } - fields, ok := ctx.Value(contextLogFieldsKey).(log.Fields) - if !ok { - fields = log.Fields{} - } - csiReqID, ok := ctx.Value(csictx.RequestIDKey).(string) - if !ok { - return fields - } - fields["RequestID"] = csiReqID - return fields -} - // GetISCSITargetsInfoFromStorage returns list of gobrick compatible iscsi targets by querying PowerStore array func GetISCSITargetsInfoFromStorage(client gopowerstore.Client, volumeApplianceID string) ([]gobrick.ISCSITargetInfo, error) { addrInfo, err := client.GetStorageISCSITargetAddresses(context.Background()) @@ -348,7 +369,7 @@ func GetISCSITargetsInfoFromStorage(client gopowerstore.Client, volumeApplianceI for _, t := range addrInfo { // volumeApplianceID will be empty in case the call is from NodeGetInfo if t.ApplianceID == volumeApplianceID || volumeApplianceID == "" { - result = append(result, gobrick.ISCSITargetInfo{Target: t.IPPort.TargetIqn, Portal: fmt.Sprintf("%s:3260", t.Address)}) + result = append(result, gobrick.ISCSITargetInfo{Target: t.IPPort.TargetIqn, Portal: fmt.Sprintf("%s:3260", t.Address), NetworkID: t.NetworkID}) } } return result, nil @@ -376,7 +397,7 @@ func GetNVMETCPTargetsInfoFromStorage(client gopowerstore.Client, volumeApplianc for _, t := range addrInfo { // volumeApplianceID will be empty in case the call is from NodeGetInfo if t.ApplianceID == volumeApplianceID || volumeApplianceID == "" { - result = append(result, gobrick.NVMeTargetInfo{Target: nvmeNQN, Portal: fmt.Sprintf("%s:4420", t.Address)}) + result = append(result, gobrick.NVMeTargetInfo{Target: nvmeNQN, Portal: fmt.Sprintf("%s:4420", t.Address), NetworkID: t.NetworkID}) } } return result, nil @@ -440,19 +461,19 @@ func ParseCIDR(externalAccessCIDR string) (string, error) { if !strings.Contains(externalAccessCIDR, "/") { // if externalAccess is a plane ip we can add /32 from our end externalAccessCIDR += "/32" - log.Debug("externalAccess after appending netMask bit:", externalAccessCIDR) + log.Debugf("externalAccess after appending netMask bit: %s", externalAccessCIDR) } ip, ipnet, err := net.ParseCIDR(externalAccessCIDR) if err != nil { return "", err } - log.Debug("Parsed CIDR:", externalAccessCIDR, "-> ip:", ip, " net:", ipnet) + log.Debugf("Parsed CIDR: %s -> ip: %v and net: %v", externalAccessCIDR, ip, ipnet) start, _ := cidr.AddressRange(ipnet) fromString, err := GetIPListWithMaskFromString(externalAccessCIDR) if err != nil { return "", err } - log.Debug("IP with Mask:", fromString) + log.Debugf("IP with Mask: %s", fromString) s := strings.Split(fromString, "/") // ExernalAccess IP consists of Starting range IP of CIDR+Mask and hence concatenating the same to remove from the array @@ -497,15 +518,16 @@ func Contains(slice []string, element string) bool { func ExternalAccessAlreadyAdded(export gopowerstore.NFSExport, externalAccess string) bool { externalAccess, _ = ParseCIDR(externalAccess) if Contains(export.RWRootHosts, externalAccess) || Contains(export.RWHosts, externalAccess) || Contains(export.RORootHosts, externalAccess) || Contains(export.ROHosts, externalAccess) { - log.Debug("ExternalAccess is already added into Host Access list on array: ", externalAccess) + log.Debugf("ExternalAccess is already added into Host Access list on array: %s ", externalAccess) return true } - log.Debug("Going to add externalAccess into Host Access list on array: ", externalAccess) + log.Debugf("Going to add externalAccess into Host Access list on array: %s", externalAccess) return false } // SetPollingFrequency reads the pollingFrequency from Env, sets default vale if ENV not found func SetPollingFrequency(ctx context.Context) int64 { + log := log.WithContext(ctx) var pollingFrequency int64 if pollRateEnv, ok := csictx.LookupEnv(ctx, EnvPodmonArrayConnectivityPollRate); ok { if pollingFrequency, _ = strconv.ParseInt(pollRateEnv, 10, 32); pollingFrequency != 0 { @@ -519,6 +541,7 @@ func SetPollingFrequency(ctx context.Context) int64 { // SetAPIPort set the port for running server func SetAPIPort(ctx context.Context) { + log := log.WithContext(ctx) if port, ok := csictx.LookupEnv(ctx, EnvPodmonAPIPORT); ok && strings.TrimSpace(port) != "" { APIPort = fmt.Sprintf(":%s", port) log.Debugf("set podmon API port to %s", APIPort) @@ -578,6 +601,19 @@ func GetTimeoutFromEnv(envvar string) (time.Duration, error) { return timeout, nil } +// GetIntFromEnv retrieves an integer value from the specified environment variable or returns an error. +func GetIntFromEnv(envvar string) (int, error) { + if valStr, ok := csictx.LookupEnv(context.Background(), envvar); ok { + val, err := strconv.Atoi(valStr) + if err != nil { + return -1, err + } + log.Infof("%s set to: %d", envvar, val) + return val, nil + } + return -1, errors.New("failed to get integer value from env") +} + // GetPodmonArrayConnectivityPollRate retrieves a timeout value for podmon node array connectivity check from env var or returns default value. func GetPodmonArrayConnectivityTimeout() time.Duration { timeout, err := GetTimeoutFromEnv(EnvPodmonArrayConnectivityTimeout) @@ -597,3 +633,45 @@ func GetPowerStoreRESTApiTimeout() time.Duration { } return timeout } + +// GetVolumeDisconnectMaxRetries retrieves the maximum number of retry attempts for volume disconnection from env var or returns the default value. +func GetVolumeDisconnectMaxRetries() int { + retries, err := GetIntFromEnv(EnvVolumeDisconnectMaxRetries) + if err != nil { + log.Debugf("failed to get max retries from env %s, using default value %d", EnvVolumeDisconnectMaxRetries, DefaultVolumeDisconnectMaxRetries) + return DefaultVolumeDisconnectMaxRetries + } + return retries +} + +// GetVolumeDisconnectRetryInterval retrieves the wait time between volume disconnection retries from env var or returns the default value. +func GetVolumeDisconnectRetryInterval() time.Duration { + timeout, err := GetTimeoutFromEnv(EnvVolumeDisconnectRetryInterval) + if err != nil { + log.Debugf("failed to get RetryInterval from env %s, using default value %d", EnvVolumeDisconnectRetryInterval, DefaultVolumeDisconnectRetryInterval) + return DefaultVolumeDisconnectRetryInterval + } + return timeout +} + +// GetVolumeDisconnectTimeout retrieves a timeout value for volume disconnection from env var or returns the default value. +func GetVolumeDisconnectTimeout() time.Duration { + timeout, err := GetTimeoutFromEnv(EnvVolumeDisconnectTimeoutSeconds) + if err != nil { + log.Debugf("failed to get timeout from env %s, using default value %d", EnvVolumeDisconnectTimeoutSeconds, DefaultVolumeDisconnectTimeout) + return DefaultVolumeDisconnectTimeout + } + return timeout +} + +// HostAlreadyPresentInNFSExport checks if the given host IP is already present in any of the NFS export host lists. +func HostAlreadyPresentInNFSExport(export gopowerstore.NFSExport, ip string) bool { + host := ip + "/255.255.255.255" + if Contains(export.ROHosts, host) || + Contains(export.RORootHosts, host) || + Contains(export.RWHosts, host) || Contains(export.RWRootHosts, host) { + log.Debug("Host IP is already present in NFS Export") + return true + } + return false +} diff --git a/pkg/identifiers/identifiers_test.go b/pkg/identifiers/identifiers_test.go index dbf94af1..6f52695c 100644 --- a/pkg/identifiers/identifiers_test.go +++ b/pkg/identifiers/identifiers_test.go @@ -27,27 +27,16 @@ import ( "testing" "time" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/mocks" identifiers "github.com/dell/csi-powerstore/v2/pkg/identifiers" - csictx "github.com/dell/gocsi/context" csiutils "github.com/dell/gocsi/utils/csi" "github.com/dell/gopowerstore" gopowerstoremock "github.com/dell/gopowerstore/mocks" - log "github.com/sirupsen/logrus" + "github.com/container-storage-interface/spec/lib/go/csi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) -func TestCustomLogger(_ *testing.T) { - log.SetLevel(log.DebugLevel) - lg := &identifiers.CustomLogger{} - ctx := context.Background() - lg.Info(ctx, "foo") - lg.Debug(ctx, "bar") - lg.Error(ctx, "spam") -} - func TestRmSockFile(t *testing.T) { sockPath := "unix:///var/run/csi/csi.sock" trimmedSockPath := "/var/run/csi/csi.sock" @@ -91,25 +80,6 @@ func TestRmSockFile(t *testing.T) { }) } -func TestSetLogFields(t *testing.T) { - t.Run("empty context", func(_ *testing.T) { - identifiers.SetLogFields(nil, log.Fields{}) - }) -} - -func TestGetLogFields(t *testing.T) { - t.Run("empty context", func(t *testing.T) { - fields := identifiers.GetLogFields(nil) - assert.Equal(t, log.Fields{}, fields) - }) - - t.Run("req id", func(t *testing.T) { - ctx := context.WithValue(context.Background(), csictx.RequestIDKey, "1") - fields := identifiers.GetLogFields(ctx) - assert.Equal(t, log.Fields{"RequestID": "1"}, fields) - }) -} - func TestGetISCSITargetsInfoFromStorage(t *testing.T) { t.Run("api error", func(t *testing.T) { e := errors.New("some error") @@ -480,7 +450,8 @@ func TestGetIPListFromString(t *testing.T) { {"InValid IP, Test 2", args{input: "10.256.1.2"}, x}, {"Valid Localhost", args{input: "https://localhost:9400"}, []string{"localhost"}}, {"Valid Domain", args{input: "https://example.com:9443"}, []string{"example.com"}}, - {"Invalid Domain", args{input: "http://mydomain."}, x}, + {"Valid CSI NodeID", args{input: "csi-node-b61220be1acc441abdd8b00e34542e5d-1.1.1.1"}, []string{"1.1.1.1"}}, + {"Valid CSI NodeID", args{input: "csi-node-tar2222.infralab.ptec-2.2.2.2"}, []string{"2.2.2.2"}}, {"Valid Multi-Segment Domain", args{input: "https://abc.example.com/page"}, []string{"abc.example.com"}}, } for _, tt := range tests { @@ -670,3 +641,190 @@ func TestGetPodmonArrayConnectivityTimeout(t *testing.T) { }) } } + +func TestGetVolumeDisconnectTimeout(t *testing.T) { + tests := []struct { + name string + expected time.Duration + setupFunc func() + teardownFunc func() + }{ + { + name: "env variable is not set", + expected: 120 * time.Second, // DefaultVolumeDisconnectTimeout + }, + { + name: "env variable is set to valid value", + expected: 45 * time.Second, + setupFunc: func() { os.Setenv("X_CSI_VOLUME_DISCONNECT_TIMEOUT_SECONDS", "45s") }, + teardownFunc: func() { os.Unsetenv("X_CSI_VOLUME_DISCONNECT_TIMEOUT_SECONDS") }, + }, + { + name: "env variable is set to invalid value", + expected: 120 * time.Second, + setupFunc: func() { os.Setenv("X_CSI_VOLUME_DISCONNECT_TIMEOUT_SECONDS", "invalid") }, + teardownFunc: func() { os.Unsetenv("X_CSI_VOLUME_DISCONNECT_TIMEOUT_SECONDS") }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setupFunc != nil { + tt.setupFunc() + defer tt.teardownFunc() + } + + actual := identifiers.GetVolumeDisconnectTimeout() + if actual != tt.expected { + t.Errorf("GetVolumeDisconnectTimeout() = %v, want %v", actual, tt.expected) + } + }) + } +} + +func TestGetVolumeDisconnectRetryInterval(t *testing.T) { + tests := []struct { + name string + expected time.Duration + setupFunc func() + teardownFunc func() + }{ + { + name: "env variable is not set", + expected: 5 * time.Second, // DefaultVolumeDisconnectRetryInterval + }, + { + name: "env variable is set to valid value", + expected: 15 * time.Second, + setupFunc: func() { os.Setenv("X_CSI_VOLUME_DISCONNECT_RETRY_INTERVAL", "15s") }, + teardownFunc: func() { os.Unsetenv("X_CSI_VOLUME_DISCONNECT_RETRY_INTERVAL") }, + }, + { + name: "env variable is set to invalid value", + expected: 5 * time.Second, + setupFunc: func() { os.Setenv("X_CSI_VOLUME_DISCONNECT_RETRY_INTERVAL", "invalid") }, + teardownFunc: func() { os.Unsetenv("X_CSI_VOLUME_DISCONNECT_RETRY_INTERVAL") }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setupFunc != nil { + tt.setupFunc() + defer tt.teardownFunc() + } + + actual := identifiers.GetVolumeDisconnectRetryInterval() + if actual != tt.expected { + t.Errorf("GetVolumeDisconnectRetryInterval() = %v, want %v", actual, tt.expected) + } + }) + } +} + +func TestGetVolumeDisconnectMaxRetries(t *testing.T) { + tests := []struct { + name string + expected int + setupFunc func() + teardownFunc func() + }{ + { + name: "env variable is not set", + expected: 5, // DefaultVolumeDisconnectMaxRetries + }, + { + name: "env variable is set to valid value", + expected: 7, + setupFunc: func() { os.Setenv("X_CSI_VOLUME_DISCONNECT_MAX_RETRIES", "7") }, + teardownFunc: func() { os.Unsetenv("X_CSI_VOLUME_DISCONNECT_MAX_RETRIES") }, + }, + { + name: "env variable is set to invalid value", + expected: 5, + setupFunc: func() { os.Setenv("X_CSI_VOLUME_DISCONNECT_MAX_RETRIES", "invalid") }, + teardownFunc: func() { os.Unsetenv("X_CSI_VOLUME_DISCONNECT_MAX_RETRIES") }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setupFunc != nil { + tt.setupFunc() + defer tt.teardownFunc() + } + + actual := identifiers.GetVolumeDisconnectMaxRetries() + if actual != tt.expected { + t.Errorf("GetVolumeDisconnectMaxRetries() = %v, want %v", actual, tt.expected) + } + }) + } +} + +func TestHostAlreadyPresentInNFSExport(t *testing.T) { + tests := []struct { + name string + export gopowerstore.NFSExport + ip string + want bool + }{ + { + name: "present_in_RWHosts", + export: gopowerstore.NFSExport{RWHosts: []string{"10.0.0.1/255.255.255.255"}}, + ip: "10.0.0.1", + want: true, + }, + { + name: "present_in_RWRootHosts", + export: gopowerstore.NFSExport{RWRootHosts: []string{"192.168.0.5/255.255.255.255"}}, + ip: "192.168.0.5", + want: true, + }, + { + name: "present_in_ROHosts", + export: gopowerstore.NFSExport{ROHosts: []string{"172.16.10.10/255.255.255.255"}}, + ip: "172.16.10.10", + want: true, + }, + { + name: "present_in_RORootHosts", + export: gopowerstore.NFSExport{RORootHosts: []string{"10.10.10.10/255.255.255.255"}}, + ip: "10.10.10.10", + want: true, + }, + { + name: "not_present_different_ip", + export: gopowerstore.NFSExport{RWHosts: []string{"10.0.0.2/255.255.255.255"}, ROHosts: []string{"10.0.0.3/255.255.255.255"}}, + ip: "10.0.0.1", + want: false, + }, + { + name: "not_present_empty_lists", + export: gopowerstore.NFSExport{}, + ip: "10.0.0.1", + want: false, + }, + { + name: "present_in_multiple_lists", + export: gopowerstore.NFSExport{RWHosts: []string{"1.2.3.4/255.255.255.255"}, RORootHosts: []string{"5.6.7.8/255.255.255.255"}}, + ip: "1.2.3.4", + want: true, + }, + { + name: "ip_with_trailing_spaces_not_present", + export: gopowerstore.NFSExport{RWHosts: []string{"8.8.8.8/255.255.255.255"}}, + ip: "8.8.8.8 ", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := identifiers.HostAlreadyPresentInNFSExport(tt.export, tt.ip) + if got != tt.want { + t.Errorf("HostAlreadyPresentInNFSExport() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/identifiers/k8sutils/k8sutils.go b/pkg/identifiers/k8sutils/k8sutils.go index f2d8ce54..a00ae308 100644 --- a/pkg/identifiers/k8sutils/k8sutils.go +++ b/pkg/identifiers/k8sutils/k8sutils.go @@ -18,110 +18,119 @@ package k8sutils import ( "context" + "encoding/json" + "errors" "fmt" "strings" + "github.com/dell/csmlog" + k8score "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) -// NodeLabelsRetrieverInterface defines the methods for retrieving Kubernetes Node Labels -type NodeLabelsRetrieverInterface interface { - BuildConfigFromFlags(masterURL, kubeconfig string) (*rest.Config, error) - InClusterConfig() (*rest.Config, error) - NewForConfig(config *rest.Config) (*kubernetes.Clientset, error) - GetNodeLabels(ctx context.Context, kubeNodeName string) (map[string]string, error) - GetNVMeUUIDs(ctx context.Context) (map[string]string, error) +type K8sClient struct { + Clientset kubernetes.Interface } -// NodeLabelsModifierInterface defines the methods for retrieving Kubernetes Node Labels -type NodeLabelsModifierInterface interface { - AddNVMeLabels(ctx context.Context, kubeNodeName string, labelKey string, labelValue []string) error +// Instantiate csmlog on a package level +var log = csmlog.GetLogger() + +// Kube Kubeclient +var Kubeclient *K8sClient + +// used for unit testing - +// allows CreateKubeClientSet to be mocked +var InClusterConfigFunc = func() (*rest.Config, error) { + return rest.InClusterConfig() } -// NodeLabelsRetrieverImpl provided the implementation for NodeLabelsRetrieverInterface -type NodeLabelsRetrieverImpl struct{} +var NewForConfigFunc = func(config *rest.Config) (kubernetes.Interface, error) { + return kubernetes.NewForConfig(config) +} -// NodeLabelsModifierImpl provides the implementation for NodeLabelsModifierInterface -type NodeLabelsModifierImpl struct{} +// CreateKubeClientSet creates kubeclient set if not created already +func CreateKubeClientSet(kubeconfig ...string) (*K8sClient, error) { + Kubeclient = &K8sClient{} -var ( - // NodeLabelsRetriever is the actual instance of NodeLabelsRetrieverInterface which is used to retrieve the node labels - NodeLabelsRetriever NodeLabelsRetrieverInterface - NodeLabelsModifier NodeLabelsModifierInterface - // Kube Clientset - Clientset kubernetes.Interface -) + config, err := InClusterConfigFunc() + if err != nil { + if len(kubeconfig) == 0 { + return nil, err + } -func init() { - NodeLabelsRetriever = new(NodeLabelsRetrieverImpl) - NodeLabelsModifier = new(NodeLabelsModifierImpl) -} + config, err = clientcmd.BuildConfigFromFlags("", kubeconfig[0]) + if err != nil { + return nil, err + } + } -// BuildConfigFromFlags is a method for building kubernetes client config -func (svc *NodeLabelsRetrieverImpl) BuildConfigFromFlags(masterURL, kubeconfig string) (*rest.Config, error) { - return clientcmd.BuildConfigFromFlags(masterURL, kubeconfig) -} + Kubeclient.Clientset, err = NewForConfigFunc(config) + if err != nil { + return nil, fmt.Errorf("failed to create Kubernetes clientset: %s", err.Error()) + } -// InClusterConfig returns a config object which uses the service account kubernetes gives to pods -func (svc *NodeLabelsRetrieverImpl) InClusterConfig() (*rest.Config, error) { - return rest.InClusterConfig() + return Kubeclient, nil } -// NewForConfig creates a new Clientset for the given config -func (svc *NodeLabelsRetrieverImpl) NewForConfig(config *rest.Config) (*kubernetes.Clientset, error) { - return kubernetes.NewForConfig(config) +func (k8s *K8sClient) GetNode(ctx context.Context, kubeNodeName string) (*k8score.Node, error) { + if k8s.Clientset == nil { + return nil, fmt.Errorf("unable to get node %q, kubernetes client is uninitialized", kubeNodeName) + } + + node, err := k8s.Clientset.CoreV1().Nodes().Get(ctx, kubeNodeName, v1.GetOptions{}) + if err != nil { + return nil, err + } + + return node, nil } // GetNodeLabels retrieves the kubernetes node object and returns its labels -func (svc *NodeLabelsRetrieverImpl) GetNodeLabels(ctx context.Context, kubeNodeName string) (map[string]string, error) { - if Clientset != nil { - node, err := Clientset.CoreV1().Nodes().Get(ctx, kubeNodeName, v1.GetOptions{}) - if err != nil { - return nil, err - } +func (k8s *K8sClient) GetNodeLabels(ctx context.Context, kubeNodeName string) (map[string]string, error) { + if k8s.Clientset == nil { + return nil, fmt.Errorf("unable to get node labels for node %q, kubernetes client is uninitialized", kubeNodeName) + } - return node.Labels, nil + node, err := k8s.GetNode(ctx, kubeNodeName) + if err != nil { + return nil, err } - return nil, nil + return node.Labels, nil } -// CreateKubeClientSet creates kubeclient set if not created already -func CreateKubeClientSet(kubeconfig string) error { - if Clientset == nil { - var config *rest.Config - var err error - if kubeconfig != "" { - config, err = NodeLabelsRetriever.BuildConfigFromFlags("", kubeconfig) - if err != nil { - return err - } - } else { - config, err = NodeLabelsRetriever.InClusterConfig() - if err != nil { - return err - } - } - // create the clientset - Clientset, err = NodeLabelsRetriever.NewForConfig(config) - if err != nil { - return err - } +// GetNodeLabels retrieves the kubernetes node object and returns its labels +func (k8s *K8sClient) SetNodeLabel(ctx context.Context, kubeNodeName string, labelKey string, labelValue string) error { + if k8s.Clientset == nil { + return fmt.Errorf("unable to get node labels for node %q, kubernetes client is uninitialized", kubeNodeName) + } + + node, err := k8s.GetNode(ctx, kubeNodeName) + if err != nil { + return err } + + // Update the node with the new labels + node.Labels[labelKey] = labelValue + _, err = k8s.Clientset.CoreV1().Nodes().Update(ctx, node, v1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update node %s labels: %v", kubeNodeName, err.Error()) + } + return nil } // AddNVMeLabels adds a hostnqn uuid label to the specified Kubernetes node -func (svc *NodeLabelsModifierImpl) AddNVMeLabels(ctx context.Context, kubeNodeName string, labelKey string, labelValue []string) error { - if Clientset == nil { - return fmt.Errorf("k8sclientset is nil") +func (k8s *K8sClient) AddNVMeLabels(ctx context.Context, kubeNodeName string, labelKey string, labelValue []string) error { + if k8s.Clientset == nil { + return fmt.Errorf("unable to add NVMe labels to node %q, kubernetes client is uninitialized", kubeNodeName) } // Get the current node - node, err := Clientset.CoreV1().Nodes().Get(ctx, kubeNodeName, v1.GetOptions{}) + node, err := k8s.GetNode(ctx, kubeNodeName) if err != nil { return fmt.Errorf("failed to get node %s: %v", kubeNodeName, err.Error()) } @@ -142,7 +151,7 @@ func (svc *NodeLabelsModifierImpl) AddNVMeLabels(ctx context.Context, kubeNodeNa // Update the node with the new labels node.Labels[labelKey] = strings.Join(uuids, ",") - _, err = Clientset.CoreV1().Nodes().Update(ctx, node, v1.UpdateOptions{}) + _, err = k8s.Clientset.CoreV1().Nodes().Update(ctx, node, v1.UpdateOptions{}) if err != nil { return fmt.Errorf("failed to update node %s labels: %v", kubeNodeName, err.Error()) } @@ -150,14 +159,14 @@ func (svc *NodeLabelsModifierImpl) AddNVMeLabels(ctx context.Context, kubeNodeNa } // GetNVMeUUIDs returns map of hosts with their hostnqn uuids -func (svc *NodeLabelsRetrieverImpl) GetNVMeUUIDs(ctx context.Context) (map[string]string, error) { +func (k8s *K8sClient) GetNVMeUUIDs(ctx context.Context) (map[string]string, error) { nodeUUIDs := make(map[string]string) - if Clientset == nil { - return nodeUUIDs, fmt.Errorf("k8sclientset is nil") + if k8s.Clientset == nil { + return nodeUUIDs, errors.New("unable to get NVMe UUIDs, kubernetes client is uninitialized") } // Retrieve the list of nodes - nodes, err := Clientset.CoreV1().Nodes().List(ctx, v1.ListOptions{}) + nodes, err := k8s.Clientset.CoreV1().Nodes().List(ctx, v1.ListOptions{}) if err != nil { return nodeUUIDs, fmt.Errorf("failed to get node list: %v", err.Error()) } @@ -173,32 +182,64 @@ func (svc *NodeLabelsRetrieverImpl) GetNVMeUUIDs(ctx context.Context) (map[strin return nodeUUIDs, nil } -// GetNodeLabels returns labels present in the k8s node -func GetNodeLabels(ctx context.Context, kubeConfigPath string, kubeNodeName string) (map[string]string, error) { - err := CreateKubeClientSet(kubeConfigPath) +func (k8s *K8sClient) GetNodeByCSINodeID(ctx context.Context, driverKey string, csiNodeID string, keyNodeID string) (*k8score.Node, error) { + if k8s.Clientset == nil { + return nil, errors.New("unable to get node, kubernetes client is uninitialized") + } + + // Retrieve the list of nodes + nodes, err := k8s.Clientset.CoreV1().Nodes().List(ctx, v1.ListOptions{}) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to get node list: %v", err.Error()) } - return NodeLabelsRetriever.GetNodeLabels(ctx, kubeNodeName) + for _, node := range nodes.Items { + if annotation, exists := node.Annotations[keyNodeID]; exists { + var nodeIDMap map[string]string + if err := json.Unmarshal([]byte(annotation), &nodeIDMap); err != nil { + continue + } + + if value, found := nodeIDMap[driverKey]; found && value == csiNodeID { + return &node, nil + } + } + } + + return nil, fmt.Errorf("failed to find a Node matching csiNodeId %s", csiNodeID) } -// AddNVMeLabels adds a hostnqn uuid label in the k8s node -func AddNVMeLabels(ctx context.Context, kubeConfigPath string, kubeNodeName string, labelKey string, labelValue []string) error { - err := CreateKubeClientSet(kubeConfigPath) - if err != nil { - return err +// ListVolumes lists all persistent volumes. +func (k8s *K8sClient) ListPersistentVolumes(ctx context.Context) (*k8score.PersistentVolumeList, error) { + if k8s.Clientset == nil { + return nil, errors.New("unable to list volumes, kubernetes client is uninitialized") } - return NodeLabelsModifier.AddNVMeLabels(ctx, kubeNodeName, labelKey, labelValue) + return k8s.Clientset.CoreV1().PersistentVolumes().List(ctx, v1.ListOptions{}) } -// GetNVMeUUIDs checks for duplicate hostnqn uuid labels in the k8s node -func GetNVMeUUIDs(ctx context.Context, kubeConfigPath string) (map[string]string, error) { - err := CreateKubeClientSet(kubeConfigPath) - if err != nil { - return map[string]string{}, err +// GetEvents gets events for the named resource of the given kind in the given namespace. +// If name is empty, it gets all events for the given kind in the given namespace. +func (k8s *K8sClient) GetEvents(ctx context.Context, kind, name, namespace string) (*k8score.EventList, error) { + if k8s.Clientset == nil { + return nil, errors.New("unable to get events, kubernetes client is uninitialized") + } + + var fieldSelector string + if name != "" { + fieldSelector = fmt.Sprintf("involvedObject.name=%s", name) } + if kind != "" { + fieldSelector = strings.Join([]string{fieldSelector, fmt.Sprintf("involvedObject.kind=%s", kind)}, ",") + } + + log.Debugf("getting events for %q %q in namespace %q", kind, name, namespace) + + return k8s.Clientset.CoreV1().Events(namespace).List(ctx, v1.ListOptions{ + FieldSelector: fieldSelector, + }) +} - return NodeLabelsRetriever.GetNVMeUUIDs(ctx) +var GetNodeByCSINodeID = func(ctx context.Context, driverKey string, csiNodeID string, keyNodeID string) (*k8score.Node, error) { + return Kubeclient.GetNodeByCSINodeID(ctx, driverKey, csiNodeID, keyNodeID) } diff --git a/pkg/identifiers/k8sutils/k8sutils_test.go b/pkg/identifiers/k8sutils/k8sutils_test.go index a6f1f575..eb229516 100644 --- a/pkg/identifiers/k8sutils/k8sutils_test.go +++ b/pkg/identifiers/k8sutils/k8sutils_test.go @@ -14,26 +14,96 @@ * limitations under the License. */ -package k8sutils_test +package k8sutils import ( "context" "errors" + "os" + "reflect" "testing" - "github.com/dell/csi-powerstore/v2/mocks" - "github.com/dell/csi-powerstore/v2/pkg/identifiers/k8sutils" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" +) + +const ( + testEventKind string = "PersistentVolume" + testNamespace string = "default" + testVolName string = "csivol-aabccdd" +) + +var ( + testPV *corev1.PersistentVolume = &corev1.PersistentVolume{ + TypeMeta: v1.TypeMeta{ + Kind: "PersistentVolume", + }, + ObjectMeta: v1.ObjectMeta{ + Name: testVolName, + Namespace: "", + }, + Spec: corev1.PersistentVolumeSpec{ + Capacity: corev1.ResourceList{ + "storage": resource.MustParse("10Gi"), + }, + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, + StorageClassName: "powerstore-block", + }, + } + testVolumeEvent *corev1.Event = &corev1.Event{ + TypeMeta: v1.TypeMeta{ + APIVersion: "v1", + Kind: "Event", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "event1", + Namespace: "default", + }, + InvolvedObject: corev1.ObjectReference{ + APIVersion: "v1", + Name: testVolName, + Kind: testEventKind, + }, + Type: corev1.EventTypeWarning, + Reason: "Minor", + } + testPodEvent *corev1.Event = &corev1.Event{ + TypeMeta: v1.TypeMeta{ + APIVersion: "v1", + Kind: "Event", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "powerstore-node-event", + Namespace: "powerstore", + }, + InvolvedObject: corev1.ObjectReference{ + Name: "powerstore-node-aabbccdd", + Kind: "Pod", + Namespace: "powerstore", + }, + Type: corev1.EventTypeNormal, + Reason: "Scheduled", + } ) func GetMockNodeWithLabels() *corev1.Node { return &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "node1", + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": "{\"node1\":\"myCsiNode\"}", + }, Labels: map[string]string{ "max-powerstore-volumes-per-node": "2", "hostnqn-uuid": "uuid1", @@ -50,152 +120,446 @@ func GetMockNodeWithoutLabels() *corev1.Node { } } -func TestUtilFunctions(t *testing.T) { - nodeLabelsRetriever := &k8sutils.NodeLabelsRetrieverImpl{} - nodeLabelsModifier := &k8sutils.NodeLabelsModifierImpl{} - t.Run("GetNodeLabels", func(t *testing.T) { - k8sutils.Clientset = fake.NewClientset(GetMockNodeWithLabels()) +func TestCreateKubeClientSet(t *testing.T) { + var tempConfigFunc func() (*rest.Config, error) // must return getInClusterConfig to its original value + var tempClientsetFunc func(config *rest.Config) (kubernetes.Interface, error) // must return getK8sClientset to its original value + + tests := []struct { + name string + before func(*testing.T) error + after func() + wantErr bool + }{ + { + name: "success: manually set InClusterConfig with mock", + before: func(_ *testing.T) error { + Kubeclient = nil // reset Clientset before each run + tempConfigFunc = InClusterConfigFunc + InClusterConfigFunc = func() (*rest.Config, error) { return &rest.Config{}, nil } + return nil + }, + after: func() { InClusterConfigFunc = tempConfigFunc }, + wantErr: false, + }, + { + name: "failure: unmocked config function", + before: func(tt *testing.T) error { + Kubeclient = nil // reset Clientset before each run + tempConfigFunc = InClusterConfigFunc + // Mock InClusterConfigFunc to return an error to simulate failure + InClusterConfigFunc = func() (*rest.Config, error) { + return nil, errors.New("unable to load in-cluster configuration") + } + // Clear KUBECONFIG to ensure fallback also fails + tt.Setenv(identifiers.EnvKubeConfigPath, "") + return nil + }, + after: func() { + InClusterConfigFunc = tempConfigFunc + }, + wantErr: true, + }, + { + name: "failure: error returned by kubernetes.NewForConfig", + before: func(_ *testing.T) error { // overrides to get past a mock and inject a failure + Kubeclient = nil // reset Clientset before each run + tempConfigFunc = InClusterConfigFunc + tempClientsetFunc = NewForConfigFunc + InClusterConfigFunc = func() (*rest.Config, error) { return &rest.Config{}, nil } + NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return nil, assert.AnError + } + return nil + }, + after: func() { // restore functions to their defaults + InClusterConfigFunc = tempConfigFunc + NewForConfigFunc = tempClientsetFunc + }, + wantErr: true, + }, + { + name: "fail to get in-cluster config and no kubeconfig provided", + before: func(tt *testing.T) error { + Kubeclient = nil + // keep InClusterConfigFunc as-is + // Clear KUBECONFIG to ensure fallback also fails + tt.Setenv(identifiers.EnvKubeConfigPath, "") + // ensure failure of InClusterConfigFunc + tt.Setenv("KUBERNETES_SERVICE_HOST", "") + return nil + }, + after: func() {}, + wantErr: true, + }, + } - labels, err := nodeLabelsRetriever.GetNodeLabels(context.Background(), "node1") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.before(t) + defer tt.after() + + _, err := CreateKubeClientSet() + + if tt.wantErr { + assert.Error(t, err) + assert.Nil(t, Kubeclient.Clientset) + } else { + assert.NoError(t, err) + assert.NotNil(t, Kubeclient.Clientset) + } + + // Reset Clientset for the second test call + Kubeclient = nil + + // Test 2: Call CreateKubeClientSet(kubeConfig) with parameters + // For the failure test case, we need to ensure this also fails + if tt.name == "failure: unmocked config function" { + // For this test case, pass an invalid kubeconfig path to ensure failure + _, err = CreateKubeClientSet("/invalid/path/to/kubeconfig") + } else { + // For other tests, use the original kubeconfig (if any) + _, err = CreateKubeClientSet(os.Getenv(identifiers.EnvKubeConfigPath)) + } + + if tt.wantErr { + assert.Error(t, err) + assert.Nil(t, Kubeclient.Clientset) + } else { + assert.NoError(t, err) + assert.NotNil(t, Kubeclient.Clientset) + } + }) + } +} + +func TestGetNode(t *testing.T) { + t.Run("GetNode success", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithoutLabels()), + } + _, err := Kubeclient.GetNode(context.Background(), "node1") assert.NoError(t, err) - assert.Equal(t, map[string]string{"max-powerstore-volumes-per-node": "2", "hostnqn-uuid": "uuid1"}, labels) }) - t.Run("GetNVMeUUIDs", func(t *testing.T) { - k8sutils.Clientset = fake.NewClientset(GetMockNodeWithLabels()) + t.Run("GetNode no client", func(t *testing.T) { + Kubeclient = &K8sClient{} - nodeUUIDs, err := nodeLabelsRetriever.GetNVMeUUIDs(context.Background()) - - assert.NoError(t, err) - assert.Equal(t, map[string]string{"node1": "uuid1"}, nodeUUIDs) + _, err := Kubeclient.GetNode(context.Background(), "node1") + assert.Error(t, err) }) - t.Run("AddNVMeLabels", func(t *testing.T) { - k8sutils.Clientset = fake.NewClientset(GetMockNodeWithoutLabels()) + t.Run("GetNode not found", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(), + } + + _, err := Kubeclient.GetNode(context.Background(), "node1") + assert.Error(t, err) + assert.Contains(t, err.Error(), "not found") + }) +} - err := nodeLabelsModifier.AddNVMeLabels(context.Background(), "node1", "hostnqn", []string{"nqn.2025-mm.nvmexpress:uuid:xxxx-yyyy-zzzz"}) +func TestGetNodeLabels(t *testing.T) { + t.Run("GetNodeLabels success", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithLabels()), + } + labels, err := Kubeclient.GetNodeLabels(context.Background(), "node1") assert.NoError(t, err) - // Assert the node labels in the fake client set - node, err := k8sutils.Clientset.CoreV1().Nodes().Get(context.Background(), "node1", metav1.GetOptions{}) - assert.NoError(t, err) - assert.Equal(t, map[string]string{"hostnqn": "xxxx-yyyy-zzzz"}, node.Labels) + assert.Equal(t, len(labels), 2) }) -} -func TestUtilFunctions_Error(t *testing.T) { - nodeLabelsRetriever := &k8sutils.NodeLabelsRetrieverImpl{} - nodeLabelsModifier := &k8sutils.NodeLabelsModifierImpl{} - t.Run("GetNodeLabels error", func(t *testing.T) { - k8sutils.Clientset = fake.NewClientset() + t.Run("GetNodeLabels success - no labels", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithoutLabels()), + } - labels, err := nodeLabelsRetriever.GetNodeLabels(context.Background(), "node1") + labels, err := Kubeclient.GetNodeLabels(context.Background(), "node1") + assert.NoError(t, err) + assert.Equal(t, len(labels), 0) + }) + t.Run("GetNodeLabels no client", func(t *testing.T) { + Kubeclient = &K8sClient{} + + _, err := Kubeclient.GetNodeLabels(context.Background(), "node1") assert.Error(t, err) - assert.Nil(t, labels) }) - t.Run("GetNVMeUUIDs error", func(t *testing.T) { - k8sutils.Clientset = nil - - _, err := nodeLabelsRetriever.GetNVMeUUIDs(context.Background()) + t.Run("GetNodeLabels not found", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(), + } + _, err := Kubeclient.GetNodeLabels(context.Background(), "node1") assert.Error(t, err) - assert.Contains(t, err.Error(), "k8sclientset is nil") + assert.Contains(t, err.Error(), "not found") }) +} - t.Run("AddNVMeLabels get error", func(t *testing.T) { - k8sutils.Clientset = fake.NewClientset() +func TestSetNodeLabel(t *testing.T) { + t.Run("SetNodeLabel success", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithLabels()), + } - err := nodeLabelsModifier.AddNVMeLabels(context.Background(), "node1", "hostnqn", []string{"nqn.2025-mm.nvmexpress:uuid:xxxx-yyyy-zzzz"}) + err := Kubeclient.SetNodeLabel(context.Background(), "node1", "topology.kubernetes.io/zone", "zone1") + assert.NoError(t, err) - assert.Error(t, err) - assert.Contains(t, err.Error(), "failed to get node") + labels, err := Kubeclient.GetNodeLabels(context.Background(), "node1") + assert.NoError(t, err) + + assert.Equal(t, labels["topology.kubernetes.io/zone"], "zone1") }) - t.Run("AddNVMeLabels error", func(t *testing.T) { - k8sutils.Clientset = nil + t.Run("SetNodeLabel no client", func(t *testing.T) { + Kubeclient = &K8sClient{} + + err := Kubeclient.SetNodeLabel(context.Background(), "", "", "") + assert.Error(t, err) + }) - err := nodeLabelsModifier.AddNVMeLabels(context.Background(), "node1", "hostnqn", []string{"nqn.2025-mm.nvmexpress:uuid:xxxx-yyyy-zzzz"}) + t.Run("SetNodeLabel not found", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(), + } + err := Kubeclient.SetNodeLabel(context.Background(), "node1", "", "") assert.Error(t, err) - assert.Contains(t, err.Error(), "k8sclientset is nil") + assert.Contains(t, err.Error(), "not found") }) } -func TestNodeLabelRetrieverAndModifier(t *testing.T) { - retrieverMock := new(mocks.NodeLabelsRetrieverInterface) - k8sutils.NodeLabelsRetriever = retrieverMock - modifierMock := new(mocks.NodeLabelsModifierInterface) - k8sutils.NodeLabelsModifier = modifierMock - - retrieverMock.On("InClusterConfig", mock.Anything).Return(nil, nil) - retrieverMock.On("BuildConfigFromFlags", mock.Anything, mock.Anything).Return(nil, nil) - retrieverMock.On("NewForConfig", mock.Anything).Return(nil, nil) +func TestGetNVMeUUIDs(t *testing.T) { + t.Run("GetNVMeUUIDs success", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithLabels()), + } - t.Run("GetNodeLabels", func(t *testing.T) { - retrieverMock.On("GetNodeLabels", mock.Anything, mock.Anything, mock.Anything).Return( - map[string]string{"max-powerstore-volumes-per-node": "2"}, nil) + nodeUUIDs, err := Kubeclient.GetNVMeUUIDs(context.Background()) + assert.NoError(t, err) + assert.Equal(t, map[string]string{"node1": "uuid1"}, nodeUUIDs) + }) - labels, err := k8sutils.GetNodeLabels(context.Background(), "", "test-node") + t.Run("GetNVMeUUIDs success - no uuids", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithoutLabels()), + } + nodeUUIDs, err := Kubeclient.GetNVMeUUIDs(context.Background()) assert.NoError(t, err) - assert.Equal(t, map[string]string{"max-powerstore-volumes-per-node": "2"}, labels) + assert.Equal(t, len(nodeUUIDs), 0) }) - t.Run("GetNVMeUUIDs", func(t *testing.T) { - retrieverMock.On("GetNVMeUUIDs", mock.Anything, mock.Anything).Return( - map[string]string{"node1": "uuid1", "node2": "uuid2"}, nil) + t.Run("GetNVMeUUIDs no client", func(t *testing.T) { + Kubeclient = &K8sClient{} - nodeUUIDs, err := k8sutils.GetNVMeUUIDs(context.Background(), "") + _, err := Kubeclient.GetNVMeUUIDs(context.Background()) + assert.Error(t, err) + }) +} +func TestAddNVMeLabels(t *testing.T) { + t.Run("AddNVMeLabels success", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithLabels()), + } + + err := Kubeclient.AddNVMeLabels(context.Background(), "node1", "newNVME", []string{"nqn.yyyy-mm.nvmexpress:uuid:xxxx-yyyy-zzzz"}) assert.NoError(t, err) - assert.Equal(t, map[string]string{"node1": "uuid1", "node2": "uuid2"}, nodeUUIDs) }) - t.Run("AddNVMeLabels", func(t *testing.T) { - modifierMock.On("AddNVMeLabels", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - - err := k8sutils.AddNVMeLabels(context.Background(), "/config/path", "test-node", "max-powerstore-volumes-per-node", []string{"2"}) + t.Run("AddNVMeLabels success - empty labels", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithoutLabels()), + } + err := Kubeclient.AddNVMeLabels(context.Background(), "node1", "newNVME", []string{"nqn.yyyy-mm.nvmexpress:uuid:xxxx-yyyy-zzzz"}) assert.NoError(t, err) }) + + t.Run("AddNVMeLabels no client", func(t *testing.T) { + Kubeclient = &K8sClient{} + + err := Kubeclient.AddNVMeLabels(context.Background(), "node1", "newNVME", []string{"nqn.yyyy-mm.nvmexpress:uuid:xxxx-yyyy-zzzz"}) + assert.Error(t, err) + }) } -func TestNodeLabelRetriever_ConfigError(t *testing.T) { - k8sutils.Clientset = nil - retrieverMock := new(mocks.NodeLabelsRetrieverInterface) - k8sutils.NodeLabelsRetriever = retrieverMock - retrieverMock.On("InClusterConfig", mock.Anything).Return(nil, errors.New("Unable to create kubeclientset")) - retrieverMock.On("BuildConfigFromFlags", mock.Anything, mock.Anything).Return(nil, errors.New("Unable to build config")) +func TestGetNodeByCSINodeID(t *testing.T) { + t.Run("GetNodeByCSINodeID success", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithLabels()), + } + + _, err := Kubeclient.GetNodeByCSINodeID(context.Background(), "node1", "myCsiNode", "csi.volume.kubernetes.io/nodeid") + assert.NoError(t, err) + }) - t.Run("GetNodeLabels error", func(t *testing.T) { - _, err := k8sutils.GetNodeLabels(context.Background(), "", "test-node") + t.Run("GetNodeByCSINodeID failed", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithoutLabels()), + } + _, err := Kubeclient.GetNodeByCSINodeID(context.Background(), "node1", "myCsiNode", "csi.volume.kubernetes.io/nodeid") assert.Error(t, err) - assert.Contains(t, err.Error(), "Unable to create kubeclientset") }) - t.Run("AddNVMeLabels error", func(t *testing.T) { - err := k8sutils.AddNVMeLabels(context.Background(), "/config/path", "test-node", "max-powerstore-volumes-per-node", []string{"2"}) + t.Run("GetNodeByCSINodeID failed - no node found", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithoutLabels()), + } + _, err := Kubeclient.GetNodeByCSINodeID(context.Background(), "invalidNode", "myCsiNode", "csi.volume.kubernetes.io/nodeid") + assert.Error(t, err) + }) + + t.Run("GetNodeByCSINodeID no client", func(t *testing.T) { + Kubeclient = &K8sClient{} + + _, err := Kubeclient.GetNodeByCSINodeID(context.Background(), "node1", "myCsiNode", "csi.volume.kubernetes.io/nodeid") assert.Error(t, err) - assert.Contains(t, err.Error(), "Unable to build config") }) } -func TestNodeLabelRetriever_CreateError(t *testing.T) { - k8sutils.Clientset = nil - retrieverMock := new(mocks.NodeLabelsRetrieverInterface) - k8sutils.NodeLabelsRetriever = retrieverMock - retrieverMock.On("InClusterConfig", mock.Anything).Return(nil, nil) - retrieverMock.On("NewForConfig", mock.Anything).Return(nil, errors.New("Unable to create kubeclientset")) +func TestK8sClient_ListPersistentVolumes(t *testing.T) { + tests := []struct { + name string // description of this test case + // Named input parameters for receiver constructor. + k8s func() *K8sClient + want *corev1.PersistentVolumeList + wantErr bool + }{ + { + name: "uninitialized kube client", + k8s: func() *K8sClient { + return &K8sClient{} + }, + want: nil, + wantErr: true, + }, + { + name: "success", + k8s: func() *K8sClient { + return &K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{testPV}...), + } + }, + want: &corev1.PersistentVolumeList{Items: []corev1.PersistentVolume{*testPV}}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + k8s := tt.k8s() + + got, gotErr := k8s.ListPersistentVolumes(context.Background()) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("ListPersistentVolumes() failed: %v", gotErr) + } + return + } + if tt.wantErr { + t.Fatal("ListPersistentVolumes() succeeded unexpectedly") + } + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ListPersistentVolumes() = %v, want %v", got, tt.want) + } + }) + } +} - t.Run("GetNVMeUUIDs error", func(t *testing.T) { - _, err := k8sutils.GetNVMeUUIDs(context.Background(), "") +func TestK8sClient_GetEvents(t *testing.T) { + type args struct { + name string + kind string + namespace string + } + tests := []struct { + name string // description of this test case + // Named input parameters for receiver constructor. + k8s func() *K8sClient + args args + want *corev1.EventList + wantErr bool + }{ + { + name: "client is uninitialized", + k8s: func() *K8sClient { return &K8sClient{} }, + args: args{ + name: testVolName, + kind: testEventKind, + namespace: "", + }, + want: nil, + wantErr: true, + }, + // NOTE: "k8s.io/client-go/kubernetes/fake" as of version v0.34.0 does not support filtering events + // based on FieldSelector, so we cannot accurately test the functionality of the GetEvents() func. + // { + // name: "without a kind", + // k8s: func() *K8sClient { + // client := fake.NewClientset([]runtime.Object{testVolumeEvent, testPodEvent}...) + // return &K8sClient{ + // Clientset: client, + // } + // }, + // args: args{ + // name: testVolName, + // kind: "", + // namespace: "", + // }, + // want: &corev1.EventList{Items: []corev1.Event{*testVolumeEvent}}, + // wantErr: false, + // }, + { + name: "gets all events in the default namespace", + k8s: func() *K8sClient { + client := fake.NewClientset([]runtime.Object{testVolumeEvent, testPodEvent}...) + return &K8sClient{ + Clientset: client, + } + }, + args: args{ + name: "", + kind: "", + namespace: "default", + }, + want: &corev1.EventList{Items: []corev1.Event{*testVolumeEvent}}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + k8s := tt.k8s() + got, gotErr := k8s.GetEvents(context.Background(), tt.args.kind, tt.args.name, tt.args.namespace) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("GetEvents() failed: %v", gotErr) + } + return + } + if tt.wantErr { + t.Fatal("GetEvents() succeeded unexpectedly") + } + if !reflect.DeepEqual(tt.want, got) { + t.Errorf("GetEvents() = %v, want %v", got, tt.want) + } + }) + } +} - assert.Error(t, err) - assert.Contains(t, err.Error(), "Unable to create kubeclientset") +func TestGetNodeByCSINodeIDVar(t *testing.T) { + t.Run("GetNodeByCSINodeIDVar success", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithLabels()), + } + + _, err := GetNodeByCSINodeID(context.Background(), "node1", "myCsiNode", "csi.volume.kubernetes.io/nodeid") + assert.NoError(t, err) }) } diff --git a/pkg/identifiers/logger.go b/pkg/identifiers/logger.go index 06e57306..f79c5a53 100644 --- a/pkg/identifiers/logger.go +++ b/pkg/identifiers/logger.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,24 +20,22 @@ package identifiers import ( "context" - - log "github.com/sirupsen/logrus" ) // CustomLogger is logger wrapper that can be passed to gopowerstore, gobrick allowing to logging context fields with each call type CustomLogger struct{} -// Info is a wrapper of logrus Info method +// Info is a wrapper of csmlog Info method func (lg *CustomLogger) Info(ctx context.Context, format string, args ...interface{}) { - log.WithFields(GetLogFields(ctx)).Infof(format, args...) + log.WithContext(ctx).Infof(format, args...) } -// Debug is a wrapper of logrus Debug method +// Debug is a wrapper of csmlog Debug method func (lg *CustomLogger) Debug(ctx context.Context, format string, args ...interface{}) { - log.WithFields(GetLogFields(ctx)).Debugf(format, args...) + log.WithContext(ctx).Debugf(format, args...) } -// Error is a wrapper of logrus Error method +// Error is a wrapper of csmlog Error method func (lg *CustomLogger) Error(ctx context.Context, format string, args ...interface{}) { - log.WithFields(GetLogFields(ctx)).Errorf(format, args...) + log.WithContext(ctx).Errorf(format, args...) } diff --git a/pkg/identity/identity_test.go b/pkg/identity/identity_test.go index 14eb2c5c..8b8e1aea 100644 --- a/pkg/identity/identity_test.go +++ b/pkg/identity/identity_test.go @@ -22,8 +22,8 @@ import ( "context" "testing" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/container-storage-interface/spec/lib/go/csi" ginkgo "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" gomega "github.com/onsi/gomega" diff --git a/pkg/interceptors/interceptors.go b/pkg/interceptors/interceptors.go index acfa8a39..ad682df8 100644 --- a/pkg/interceptors/interceptors.go +++ b/pkg/interceptors/interceptors.go @@ -26,19 +26,19 @@ import ( "sync" "time" - "github.com/akutz/gosync" - "github.com/container-storage-interface/spec/lib/go/csi" controller "github.com/dell/csi-powerstore/v2/pkg/controller" "github.com/dell/csi-powerstore/v2/pkg/identifiers" "github.com/dell/gocsi/middleware/serialvolume" + "github.com/akutz/gosync" + "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" + "github.com/dell/csmlog" csictx "github.com/dell/gocsi/context" mwtypes "github.com/dell/gocsi/middleware/serialvolume/lockprovider" - log "github.com/sirupsen/logrus" xctx "golang.org/x/net/context" "github.com/dell/csi-metadata-retriever/retriever" @@ -46,6 +46,9 @@ import ( "github.com/kubernetes-csi/csi-lib-utils/metrics" ) +// Instantial csmlog on a package level +var log = csmlog.GetLogger() + type rewriteRequestIDInterceptor struct{} func (r *rewriteRequestIDInterceptor) handleServer(ctx context.Context, req interface{}, @@ -145,6 +148,7 @@ func NewCustomSerialLock(mode string) grpc.UnaryServerInterceptor { } func (i *interceptor) createMetadataRetrieverClient(ctx context.Context) { + log := log.WithContext(ctx) metricsManager := metrics.NewCSIMetricsManagerWithOptions("csi-metadata-retriever", metrics.WithProcessStartTime(false), metrics.WithSubsystem(metrics.SubsystemSidecar)) @@ -161,7 +165,7 @@ func (i *interceptor) createMetadataRetrieverClient(ctx context.Context) { i.opts.MetadataSidecarClient = retrieverClient } else { - log.Warn("env var not found: ", identifiers.EnvMetadataRetrieverEndpoint) + log.Warnf("env var not found: %s", identifiers.EnvMetadataRetrieverEndpoint) } } @@ -208,6 +212,7 @@ func (i *interceptor) nodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag func (i *interceptor) createVolume(ctx context.Context, req *csi.CreateVolumeRequest, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler, ) (res interface{}, resErr error) { + log := log.WithContext(ctx) lock, err := i.opts.locker.GetLockWithID(ctx, req.Name) if err != nil { return nil, err diff --git a/pkg/interceptors/interceptors_test.go b/pkg/interceptors/interceptors_test.go index 2f82767b..17edfc86 100644 --- a/pkg/interceptors/interceptors_test.go +++ b/pkg/interceptors/interceptors_test.go @@ -26,13 +26,13 @@ import ( "testing" "time" - "github.com/akutz/gosync" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-metadata-retriever/retriever" controller "github.com/dell/csi-powerstore/v2/pkg/controller" "github.com/dell/csi-powerstore/v2/pkg/identifiers" csictx "github.com/dell/gocsi/context" lockprovider "github.com/dell/gocsi/middleware/serialvolume/lockprovider" + "github.com/akutz/gosync" + "github.com/container-storage-interface/spec/lib/go/csi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "google.golang.org/grpc" diff --git a/pkg/monitor/event.go b/pkg/monitor/event.go new file mode 100644 index 00000000..bc3da3e7 --- /dev/null +++ b/pkg/monitor/event.go @@ -0,0 +1,274 @@ +/* + * + * Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package monitor + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/dell/csi-powerstore/v2/pkg/array" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/k8sutils" + "github.com/dell/csmlog" + "github.com/dell/gopowerstore" + + csictx "github.com/dell/gocsi/context" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + typedv1core "k8s.io/client-go/kubernetes/typed/core/v1" + + "k8s.io/client-go/tools/record" +) + +type IMonitorService interface { + // Reads the array secret from the filepath and populates array.Locker + // with array info and API clients. + UpdateArrays(arrayConfigFilepath string, fs fs.Interface) error + // Starts the service, polling for PowerStore Alerts and Events + // every pollPeriod. + Start(ctx context.Context, pollPeriod time.Duration) +} + +// Service represents the volume event monitoring service +type Service struct { + EventRecorder record.EventRecorderLogger + EventBroadcaster record.EventBroadcaster + + array.Locker + + kubeclient *k8sutils.K8sClient +} + +type EventContent struct { + // LastRecord is a reference to the most recent Kubernetes + // event for a specific resource + LatestRecord *corev1.Event +} + +// PersistentVolumeEvent relates a Persistent Volume struct to its +// most recent Kubernetes event. +type PersistentVolumeEvent struct { + EventContent + + Volume corev1.PersistentVolume +} + +// Instantiate csmlog on a package level +var log = csmlog.GetLogger() + +const ( + timeFormat = "2006-01-02T15:04:05Z" + VolumeResourceType = "volume" +) + +// NewMonitorService creates a new monitor service. +// The Kubernetes client is created using the environment var, identifiers.EnvKubeConfigPath, +// or in-cluster config, and used to create the EventRecorder and EventBroadcaster. +func NewMonitorService(ctx context.Context) (IMonitorService, error) { + kubeConfigPath, _ := csictx.LookupEnv(ctx, identifiers.EnvKubeConfigPath) + kubeclient, err := k8sutils.CreateKubeClientSet(kubeConfigPath) + if err != nil { + return nil, fmt.Errorf("failed to create Kubernetes API client for the monitor service: %s", err.Error()) + } + + eventRecorder, eventBroadcaster, err := newEventRecorder(kubeclient) + if err != nil { + return nil, err + } + + return &Service{ + EventRecorder: eventRecorder, + EventBroadcaster: eventBroadcaster, + + kubeclient: kubeclient, + }, nil +} + +func newEventRecorder(kubeclient *k8sutils.K8sClient) (record.EventRecorderLogger, record.EventBroadcaster, error) { + eventBroadcaster := record.NewBroadcaster() + + eventBroadcaster.StartRecordingToSink(&typedv1core.EventSinkImpl{Interface: kubeclient.Clientset.CoreV1().Events("")}) + + scheme := runtime.NewScheme() + err := corev1.AddToScheme(scheme) + if err != nil { + return nil, nil, err + } + + eventRecorder := eventBroadcaster.NewRecorder(scheme, corev1.EventSource{Component: identifiers.Name}) + + return eventRecorder, eventBroadcaster, nil +} + +// Start starts monitoring volumes and logging kubernetes events for alerts +// associated with Persistent Volumes backed by the PowerStore array(s). +func (s *Service) Start(ctx context.Context, pollPeriod time.Duration) { + log.Infof("[Monitor] starting event monitor with poll period %s", pollPeriod) + ticker := time.NewTicker(pollPeriod).C + + defer s.EventBroadcaster.Shutdown() + // assumes events have already been recorded for pre-existing volumes + // in order to avoid recording very old events. + lastCheck := time.Now() + for { + select { + case <-ctx.Done(): + log.Debugf("[Monitor] context expired for volume event monitor: %s", ctx.Err()) + return + case now := <-ticker: + s.monitorSince(lastCheck) + lastCheck = now + } + } +} + +// monitorSince queries all PowerStore arrays for alerts that +// have occurred between now and the lastTime the function was run, and processes +// any new alerts, creating kubernetes events, as needed. +func (s *Service) monitorSince(lastTime time.Time) { + log.Debugf("[Monitor] Getting alerts and events since %s", lastTime) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + for _, arr := range s.Locker.Arrays() { + alerts := []gopowerstore.Alert{} + + log.Debugf("[Monitor] Getting latest alerts for array %q", arr.GlobalID) + pageIndex := 0 + for { + // get alerts since the last time they were read, + // reading until there are no more pages of alerts available + alertsResp, err := arr.GetClient().GetAlerts(ctx, gopowerstore.GetAlertsOpts{ + Queries: map[string]string{ + "generated_timestamp": fmt.Sprintf("gte.%s", lastTime.Format(timeFormat)), + "order": "generated_timestamp.asc", + }, + RequestPagination: gopowerstore.RequestPagination{ + PageSize: 1000, + StartIndex: pageIndex, + }, + }) + if err != nil { + log.Errorf("[Monitor] failed to get alerts for array %q: %s", arr.GlobalID, err) + return + } + alerts = append(alerts, alertsResp.Alerts...) + if alertsResp.Pagination.Next == 0 { + break + } + pageIndex = alertsResp.Pagination.Next + } + + log.Debugf("[Monitor] got alerts: %v", alerts) + s.processVolumeObjectEvents(ctx, alerts) + } +} + +// processVolumeObjectEvents steps through the provided PowerStore alerts, +// looking for alerts associated with Persistent Volumes (PVs) in the cluster. +// If alerts are found for a given PV, an event is logged with the event recorder. +// Alerts passed in should be sorted in ascending order to ensure they are recorded +// in the order they occurred on the PowerStore array. +func (s *Service) processVolumeObjectEvents(ctx context.Context, alerts gopowerstore.Alerts) { + persistentVolumes := s.createVolumeMap(ctx) + + for _, alert := range alerts { + // currently only monitoring alerts for "volume" type + if alert.ResourceType != VolumeResourceType { + continue + } + + event, found := persistentVolumes[alert.ResourceName] + if !found { + // skip recording if the PowerStore alert does not belong to any of the Persistent Volumes + continue + } + + // Do not record the same event twice. + if event.LatestRecord != nil && event.LatestRecord.Message == alert.Description { + continue + } + + eventType := corev1.EventTypeWarning + if strings.EqualFold(alert.Severity, "info") { + eventType = corev1.EventTypeNormal + } + log.Infof("[Monitor] Alert is active for volume %q. recording event: %s", event.Volume.Name, alert.Description) + s.EventRecorder.Event(&event.Volume, eventType, alert.Severity, alert.Description) + } +} + +// createVolumeMap returns a map of Persistent Volume names to their PersistentVolumeEvents, +// where a PersistentVolumeEvent contains a reference to the PersistentVolume and the most +// recently recorded event for that volume (if one exists). +func (s *Service) createVolumeMap(ctx context.Context) map[string]PersistentVolumeEvent { + volumes, err := s.kubeclient.ListPersistentVolumes(ctx) + if err != nil { + log.Errorf("[Monitor] failed to get persistent volumes: %s", err.Error()) + return nil + } + + log.Debugf("[Monitor] got persistent volumes: %v", volumes.Items) + + // Create map to easily navigate through volumes. + volumesMap := make(map[string]PersistentVolumeEvent) + for _, volume := range volumes.Items { + latestObjectEvent := s.getLatestK8sEvent(ctx, volume.Name, volume.Namespace, "PersistentVolume") + log.Debugf("[Monitor] got latest event for volume %q: %v", volume.Name, latestObjectEvent) + volumesMap[volume.Name] = PersistentVolumeEvent{ + Volume: volume, + EventContent: EventContent{ + LatestRecord: latestObjectEvent, + }, + } + } + + return volumesMap +} + +// getLatestK8sEvent queries the cluster for all events related to the resource, "kind", of the given +// name, in the namespace provided, and returns the latest event. +func (s *Service) getLatestK8sEvent(ctx context.Context, name, namespace, kind string) *corev1.Event { + events, err := s.kubeclient.GetEvents(ctx, kind, name, namespace) + if err != nil { + log.Errorf("[Monitor] failed to get kubernetes events for %s %q: %s", kind, name, err.Error()) + return nil + } + log.Debugf("[Monitor] got kubernetes events for %s %q: %v", kind, name, events.Items) + + if len(events.Items) == 0 { + return nil + } + + // Retrieval of Events from a k8sObject is not guaranteed to be sorted. + latestEvent := events.Items[0] + for _, event := range events.Items { + if event.LastTimestamp.After(latestEvent.LastTimestamp.Time) { + latestEvent = event + } + } + + log.Debugf("[Monitor] latest k8s event for %s %q: description: %s", kind, name, latestEvent.Message) + + return &latestEvent +} diff --git a/pkg/monitor/event_test.go b/pkg/monitor/event_test.go new file mode 100644 index 00000000..803e3f79 --- /dev/null +++ b/pkg/monitor/event_test.go @@ -0,0 +1,778 @@ +/* + * + * Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package monitor + +import ( + "context" + "errors" + "fmt" + "os" + "reflect" + "strings" + "testing" + "time" + + "github.com/dell/csi-powerstore/v2/pkg/array" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/k8sutils" + csictx "github.com/dell/gocsi/context" + "github.com/dell/gopowerstore" + "github.com/dell/gopowerstore/api" + "github.com/dell/gopowerstore/mocks" + "github.com/stretchr/testify/mock" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" +) + +const ( + testVolName string = "csi-test-vol" + testVolGroupName string = "test-vol-group" + testNamespace string = "test" + testMessageWarning string = "this is a test of the emergency alert system" + testMessageNormal string = "everything is normal" + eventMessageTypeWarning string = "Warning" + eventMessageTypeNormal string = "Normal" + alertSeverityInfo string = "Info" + alertSeverityMinor string = "Minor" + alertSeverityMajor string = "Major" + alertSeverityCritical string = "Critical" +) + +var ( + testTime time.Time = time.Now() + + testVolumeEventOldest *corev1.Event = &corev1.Event{ + TypeMeta: v1.TypeMeta{ + Kind: "PersistentVolume", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "event1", + Namespace: testNamespace, + }, + InvolvedObject: corev1.ObjectReference{ + Name: testVolName, + Namespace: testNamespace, + }, + Type: corev1.EventTypeWarning, + LastTimestamp: v1.Time{ + Time: testTime.Add(-3 * time.Minute), + }, + Message: testMessageWarning, + } + testVolumeEventLatest *corev1.Event = &corev1.Event{ + TypeMeta: v1.TypeMeta{ + Kind: "PersistentVolume", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "event3", + Namespace: testNamespace, + }, + InvolvedObject: corev1.ObjectReference{ + Name: testVolName, + Namespace: testNamespace, + }, + Type: corev1.EventTypeWarning, + LastTimestamp: v1.Time{ + Time: testTime.Add(-1 * time.Minute), + }, + Message: testMessageWarning, + } + testVolumeEventMiddle *corev1.Event = &corev1.Event{ + TypeMeta: v1.TypeMeta{ + Kind: "PersistentVolume", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "event2", + Namespace: testNamespace, + }, + InvolvedObject: corev1.ObjectReference{ + Name: testVolName, + Namespace: testNamespace, + }, + Type: corev1.EventTypeWarning, + LastTimestamp: v1.Time{ + Time: testTime.Add(-2 * time.Minute), + }, + Message: testMessageWarning, + } + testVolumeEventNormal *corev1.Event = &corev1.Event{ + TypeMeta: v1.TypeMeta{ + Kind: "PersistentVolume", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "event2", + Namespace: testNamespace, + }, + InvolvedObject: corev1.ObjectReference{ + Name: testVolName, + Namespace: testNamespace, + }, + Type: corev1.EventTypeNormal, + LastTimestamp: v1.Time{ + Time: testTime.Add(-2 * time.Minute), + }, + Message: testMessageNormal, + } + + testVolume *corev1.PersistentVolume = &corev1.PersistentVolume{ + ObjectMeta: v1.ObjectMeta{ + Name: testVolName, + Namespace: testNamespace, + }, + TypeMeta: v1.TypeMeta{ + Kind: "PersistentVolume", + }, + Status: corev1.PersistentVolumeStatus{ + Phase: corev1.VolumeBound, + }, + } + + testEvents []runtime.Object = []runtime.Object{testVolumeEventMiddle, testVolumeEventOldest, testVolumeEventLatest} +) + +func TestNewService(t *testing.T) { + tests := []struct { + name string // description of this test case + init func(context.Context, *testing.T) + cleanup func() + want *Service + wantErr bool + }{ + { + name: "fail to create kubeclient", + init: func(ctx context.Context, tt *testing.T) { + k8sutils.Kubeclient = nil + tt.Setenv(identifiers.EnvKubeConfigPath, "") + err := csictx.Setenv(ctx, identifiers.EnvKubeConfigPath, "") + if err != nil { + tt.Fatalf("failed to overwrite kubeconfig path: %v", err) + } + + tempNewForConfigFunc := k8sutils.NewForConfigFunc + k8sutils.NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return nil, errors.New("new for config error") + } + tt.Cleanup(func() { + k8sutils.NewForConfigFunc = tempNewForConfigFunc + }) + }, + want: nil, + wantErr: true, + }, + { + name: "success", + init: func(ctx context.Context, tt *testing.T) { + kubeconfigPath, err := createFakeKubeconfig(t) + if err != nil { + tt.Fatalf("failed to create fake kubeconfig for test: %s", err.Error()) + return + } + if err := csictx.Setenv(ctx, identifiers.EnvKubeConfigPath, kubeconfigPath); err != nil { + tt.Fatalf("failed to add kubeconfig variable to context: %s", err.Error()) + return + } + }, + want: &Service{}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + tt.init(ctx, t) + got, gotErr := NewMonitorService(ctx) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("NewService() failed: %v", gotErr) + } + return + } + if tt.wantErr { + t.Fatal("NewService() succeeded unexpectedly") + } + + if tt.want == nil && got != nil { + t.Errorf("NewService() = %v, want %v", got, tt.want) + } + }) + } +} + +func createFakeKubeconfig(t *testing.T) (kubeconfigPath string, err error) { + fakeConfig := ` +apiVersion: v1 +clusters: +- cluster: + server: https://localhost:8080 + name: foo-cluster +contexts: +- context: + cluster: foo-cluster + user: foo-user + namespace: bar + name: foo-context +current-context: foo-context +kind: Config +users: +- name: foo-user +` + tmpKubeconfigDir := t.TempDir() + tmpfile, err := os.CreateTemp(tmpKubeconfigDir, "kubeconfig") + if err != nil { + return "", fmt.Errorf("failed to create temp kubeconfig file: %s", err.Error()) + } + if err := os.WriteFile(tmpfile.Name(), []byte(fakeConfig), 0o600); err != nil { + return "", fmt.Errorf("failed to write config to the kubeconfig file: %s", err.Error()) + } + return tmpfile.Name(), nil +} + +func TestService_Start(t *testing.T) { + defaultArray := &array.PowerStoreArray{ + Endpoint: "my-powerstore.com/api/rest", + GlobalID: "gid1", + Username: "user", + Password: "password", + BlockProtocol: identifiers.ISCSITransport, + Insecure: true, + IsDefault: true, + } + type params struct { + pollPeriod time.Duration + ctxTimeout time.Duration + } + type fields struct { + service *Service + } + tests := []struct { + name string + fields fields + params params + getArrays func() map[string]*array.PowerStoreArray + }{ + { + name: "context timeout", + fields: fields{ + service: &Service{ + kubeclient: &k8sutils.K8sClient{ + Clientset: fake.NewClientset(), + }, + EventRecorder: record.NewFakeRecorder(0), + EventBroadcaster: record.NewBroadcasterForTests(0 * time.Second), + }, + }, + params: params{ + pollPeriod: 10 * time.Second, + // timeout being less than polling period will ensure + // the context is canceled before the initial poll. + ctxTimeout: 100 * time.Millisecond, + }, + getArrays: func() map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + defaultArray.GlobalID: defaultArray, + } + }, + }, + { + name: "a single monitor loop execution", + fields: fields{ + service: &Service{ + kubeclient: &k8sutils.K8sClient{ + Clientset: fake.NewClientset(), + }, + EventRecorder: record.NewFakeRecorder(0), + EventBroadcaster: record.NewBroadcasterForTests(0 * time.Second), + }, + }, + params: params{ + pollPeriod: 10 * time.Millisecond, + // give enough time to run the request + // but ensure the context is canceled after the first run + ctxTimeout: 11 * time.Millisecond, + }, + getArrays: func() map[string]*array.PowerStoreArray { + client := mocks.NewClient(t) + client.On("GetAlerts", mock.Anything, mock.Anything).Return(&gopowerstore.GetAlertsResponse{}, nil) + + defaultArray.Client = client + return map[string]*array.PowerStoreArray{ + defaultArray.GlobalID: defaultArray, + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(_ *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), tt.params.ctxTimeout) + defer cancel() + + s := tt.fields.service + s.SetArrays(tt.getArrays()) + s.SetDefaultArray(defaultArray) + + // it may appear nothing is being tested here, but + // the tests will pass or fail based on whether the mocked + // functions are called. + s.Start(ctx, tt.params.pollPeriod) + }) + } +} + +func TestService_monitorSince(t *testing.T) { + defaultArray := &array.PowerStoreArray{ + Endpoint: "primary.my-powerstore.com/api/rest", + GlobalID: "gid1", + Username: "user", + Password: "password", + BlockProtocol: identifiers.ISCSITransport, + Insecure: true, + IsDefault: true, + } + secondaryArray := &array.PowerStoreArray{ + Endpoint: "secondary.my-powerstore.com/api/rest", + GlobalID: "gid2", + Username: "user", + Password: "password", + BlockProtocol: identifiers.ISCSITransport, + Insecure: true, + IsDefault: false, + } + tests := []struct { + name string + getArrays func() map[string]*array.PowerStoreArray + getClient func(time.Time) gopowerstore.Client + }{ + { + name: "query array without issues", + getArrays: func() map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + defaultArray.GlobalID: defaultArray, + } + }, + getClient: func(timeStamp time.Time) gopowerstore.Client { + client := mocks.NewClient(t) + + client.On("GetAlerts", mock.Anything, gopowerstore.GetAlertsOpts{ + RequestPagination: gopowerstore.RequestPagination{ + PageSize: 1000, + StartIndex: 0, + }, + Queries: map[string]string{ + "generated_timestamp": fmt.Sprintf("gte.%s", timeStamp.Format(timeFormat)), + "order": "generated_timestamp.asc", + }, + }).Return(&gopowerstore.GetAlertsResponse{}, nil) + + return client + }, + }, + { + name: "query all arrays", + getArrays: func() map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + defaultArray.GlobalID: defaultArray, + secondaryArray.GlobalID: secondaryArray, + } + }, + getClient: func(timestamp time.Time) gopowerstore.Client { + client := mocks.NewClient(t) + client.On("GetAlerts", mock.Anything, gopowerstore.GetAlertsOpts{ + RequestPagination: gopowerstore.RequestPagination{ + PageSize: 1000, + StartIndex: 0, + }, + Queries: map[string]string{ + "generated_timestamp": fmt.Sprintf("gte.%s", timestamp.Format(timeFormat)), + "order": "generated_timestamp.asc", + }, + }).Return(&gopowerstore.GetAlertsResponse{}, nil) + + return client + }, + }, + { + name: "query paginated data", + getArrays: func() map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + defaultArray.GlobalID: defaultArray, + secondaryArray.GlobalID: secondaryArray, + } + }, + getClient: func(timestamp time.Time) gopowerstore.Client { + client := mocks.NewClient(t) + client.On("GetAlerts", mock.Anything, gopowerstore.GetAlertsOpts{ + RequestPagination: gopowerstore.RequestPagination{ + PageSize: 1000, + StartIndex: 0, + }, + Queries: map[string]string{ + "generated_timestamp": fmt.Sprintf("gte.%s", timestamp.Format(timeFormat)), + "order": "generated_timestamp.asc", + }, + }).Return(&gopowerstore.GetAlertsResponse{ + AlertsResponseMeta: gopowerstore.AlertsResponseMeta{ + RespMeta: api.RespMeta{ + Pagination: api.PaginationInfo{ + First: 0, + Last: 999, + Next: 1000, + Total: 2000, + IsPaginate: true, + }, + }, + }, + Alerts: gopowerstore.Alerts{}, + }, nil) + client.On("GetAlerts", mock.Anything, gopowerstore.GetAlertsOpts{ + RequestPagination: gopowerstore.RequestPagination{ + PageSize: 1000, + StartIndex: 1000, + }, + Queries: map[string]string{ + "generated_timestamp": fmt.Sprintf("gte.%s", timestamp.Format(timeFormat)), + "order": "generated_timestamp.asc", + }, + }).Return(&gopowerstore.GetAlertsResponse{ + AlertsResponseMeta: gopowerstore.AlertsResponseMeta{ + RespMeta: api.RespMeta{ + Pagination: api.PaginationInfo{ + First: 1000, + Last: 1999, + Next: 0, + Total: 2000, + IsPaginate: true, + }, + }, + }, + Alerts: gopowerstore.Alerts{}, + }, nil) + + return client + }, + }, + { + name: "error when querying for alerts", + getArrays: func() map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + defaultArray.GlobalID: defaultArray, + } + }, + getClient: func(timeStamp time.Time) gopowerstore.Client { + client := mocks.NewClient(t) + client.On("GetAlerts", mock.Anything, gopowerstore.GetAlertsOpts{ + RequestPagination: gopowerstore.RequestPagination{ + PageSize: 1000, + StartIndex: 0, + }, + Queries: map[string]string{ + "generated_timestamp": fmt.Sprintf("gte.%s", timeStamp.Format(timeFormat)), + "order": "generated_timestamp.asc", + }, + }).Return(nil, errors.New("foo error")) + + return client + }, + }, + { + name: "error when querying for events", + getArrays: func() map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + defaultArray.GlobalID: defaultArray, + } + }, + getClient: func(timeStamp time.Time) gopowerstore.Client { + client := mocks.NewClient(t) + + client.On("GetAlerts", mock.Anything, gopowerstore.GetAlertsOpts{ + RequestPagination: gopowerstore.RequestPagination{ + PageSize: 1000, + StartIndex: 0, + }, + Queries: map[string]string{ + "generated_timestamp": fmt.Sprintf("gte.%s", timeStamp.Format(timeFormat)), + "order": "generated_timestamp.asc", + }, + }).Return(&gopowerstore.GetAlertsResponse{}, nil) + + return client + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(_ *testing.T) { + s := &Service{ + kubeclient: &k8sutils.K8sClient{ + Clientset: fake.NewClientset(), + }, + EventRecorder: record.NewFakeRecorder(0), + EventBroadcaster: record.NewBroadcasterForTests(0 * time.Second), + } + lastTime := time.Now().Add(1 * time.Minute) + + arrays := tt.getArrays() + for _, arr := range arrays { + arr.Client = tt.getClient(lastTime) + } + s.SetArrays(arrays) + s.SetDefaultArray(defaultArray) + + // it may appear nothing is being tested here, but + // the test will confirm the expected mock functions have + // been called. + s.monitorSince(lastTime) + }) + } +} + +func TestService_processVolumeObjectEvents(t *testing.T) { + tests := []struct { + name string + alerts gopowerstore.Alerts + clusterResources []runtime.Object + want string + }{ + { + name: "record a new event from an alert", + alerts: gopowerstore.Alerts{ + { + ResourceType: VolumeResourceType, + ResourceName: testVolume.Name, + Description: testMessageWarning, + Severity: alertSeverityMinor, + }, + }, + clusterResources: []runtime.Object{testVolume}, + want: strings.Join([]string{eventMessageTypeWarning, alertSeverityMinor, testMessageWarning}, " "), + }, + { + name: "alert resource type is not a volume", + alerts: gopowerstore.Alerts{ + { + ResourceType: "volume_group", + Description: testMessageWarning, + }, + }, + clusterResources: []runtime.Object{testVolume}, + want: "", + }, + { + name: "volume is not being monitored by the driver", + alerts: gopowerstore.Alerts{ + { + ResourceName: "csivol-foo", // should not be known to the fake kubeclient + ResourceType: VolumeResourceType, + Description: testMessageWarning, + }, + }, + clusterResources: []runtime.Object{testVolume}, + want: "", + }, + { + name: "volume alert has already been submitted to event recorder", + alerts: gopowerstore.Alerts{ + { + ResourceName: testVolume.Name, + ResourceType: VolumeResourceType, + Description: testMessageWarning, + }, + }, + clusterResources: []runtime.Object{testVolume, testVolumeEventLatest}, + want: "", + }, + { + name: "volume has a new alert", + alerts: gopowerstore.Alerts{ + { + ResourceName: testVolume.Name, + ResourceType: VolumeResourceType, + Description: "this is the second test", + Severity: alertSeverityInfo, + }, + }, + clusterResources: []runtime.Object{testVolume, testVolumeEventLatest}, + want: strings.Join([]string{eventMessageTypeNormal, alertSeverityInfo, "this is the second test"}, " "), + }, + { + name: "no event updates from the array", + alerts: gopowerstore.Alerts{}, + clusterResources: []runtime.Object{testVolume, testVolumeEventLatest}, + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeRecorder := record.NewFakeRecorder(10) + s := &Service{ + kubeclient: &k8sutils.K8sClient{ + Clientset: fake.NewClientset(tt.clusterResources...), + }, + EventBroadcaster: record.NewBroadcaster(), + EventRecorder: fakeRecorder, + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + go func() { + defer cancel() + s.processVolumeObjectEvents(ctx, tt.alerts) + }() + + for { + select { + case <-ctx.Done(): + return + case event := <-fakeRecorder.Events: + if tt.want != event { + t.Errorf("processVolumeObjectEvents() = %v, want: %v", event, tt.want) + } + } + } + }) + } +} + +func TestService_CreateVolumeMap(t *testing.T) { + type fields struct { + s *Service + } + tests := []struct { + name string + fields fields + want map[string]PersistentVolumeEvent + }{ + { + name: "fail to list volumes", + fields: fields{ + s: &Service{ + // client is nil and will error + kubeclient: &k8sutils.K8sClient{}, + }, + }, + want: nil, + }, + { + name: "get volume map", + fields: fields{ + s: &Service{ + kubeclient: &k8sutils.K8sClient{ + Clientset: fake.NewClientset(append(testEvents, testVolume)...), + }, + }, + }, + want: map[string]PersistentVolumeEvent{ + testVolName: { + Volume: *testVolume, + EventContent: EventContent{ + LatestRecord: testVolumeEventLatest, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := tt.fields.s + + got := s.createVolumeMap(context.Background()) + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("CreateVolumeMap() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestService_GetLastK8sEvents(t *testing.T) { + type params struct { + name string + namespace string + kind string + } + type fields struct { + s *Service + } + tests := []struct { + name string + fields fields + params params + want *corev1.Event + }{ + { + name: "no kubeclient", + fields: fields{ + s: &Service{ + kubeclient: &k8sutils.K8sClient{}, + }, + }, + params: params{ + name: "", + namespace: "", + kind: "", + }, + want: nil, + }, + { + name: "no events", + fields: fields{ + s: &Service{ + kubeclient: &k8sutils.K8sClient{ + // init client without any resources + Clientset: fake.NewClientset(), + }, + }, + }, + params: params{ + name: "", + namespace: "", + kind: "", + }, + want: nil, + }, + { + name: "get the most recent event", + fields: fields{ + s: &Service{ + kubeclient: &k8sutils.K8sClient{ + Clientset: fake.NewClientset(testEvents...), + }, + }, + }, + params: params{ + name: testVolName, + namespace: "test", + kind: "PersistentVolume", + }, + want: testVolumeEventLatest, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := tt.fields.s + got := s.getLatestK8sEvent(context.Background(), tt.params.name, tt.params.namespace, tt.params.kind) + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetLastK8sEvents() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/node/acl.go b/pkg/node/acl.go index 5f334935..f647c59b 100644 --- a/pkg/node/acl.go +++ b/pkg/node/acl.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2022 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2022-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,6 @@ import ( "strings" "github.com/dell/gopowerstore" - log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -41,6 +40,7 @@ type NFSv4ACLsInterface interface { type NFSv4ACLs struct{} func validateAndSetACLs(ctx context.Context, s NFSv4ACLsInterface, nasName string, client gopowerstore.Client, acls string, dir string) (bool, error) { + log := log.WithContext(ctx) aclsConfigured := false if nfsv4ACLs(acls) { if isNfsv4Enabled(ctx, client, nasName) { @@ -89,6 +89,7 @@ func (n *NFSv4ACLs) SetNfsv4Acls(acls string, dir string) error { } func isNfsv4Enabled(ctx context.Context, client gopowerstore.Client, nasName string) bool { + log := log.WithContext(ctx) nfsv4Enabled := false nas, err := gopowerstore.Client.GetNASByName(client, ctx, nasName) if err == nil { diff --git a/pkg/node/base.go b/pkg/node/base.go index ef5a5ee2..c4ebe4c1 100644 --- a/pkg/node/base.go +++ b/pkg/node/base.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,14 +29,13 @@ import ( "strconv" "strings" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/pkg/identifiers" "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" - "github.com/dell/csm-sharednfs/nfs" + "github.com/dell/csmlog" "github.com/dell/gobrick" csictx "github.com/dell/gocsi/context" "github.com/dell/gofsutil" - log "github.com/sirupsen/logrus" + "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -78,6 +77,7 @@ type NVMEConnector interface { type FcConnector interface { ConnectVolume(ctx context.Context, info gobrick.FCVolumeInfo) (gobrick.Device, error) DisconnectVolumeByDeviceName(ctx context.Context, name string) error + DisconnectVolumeByWWN(ctx context.Context, wwn string) error GetInitiatorPorts(ctx context.Context) ([]string, error) } @@ -141,7 +141,7 @@ func getNodeOptions() Opts { if v, ok := csictx.LookupEnv(ctx, n); ok { b, err := strconv.ParseBool(v) if err != nil { - log.WithField(n, v).Debug("invalid boolean value. defaulting to false") + log.WithFields(csmlog.Fields{n: v}).Debug("invalid boolean value. defaulting to false") return false } return b @@ -212,20 +212,17 @@ func getStagedDev(ctx context.Context, stagePath string, fs fs.Interface) (strin } func getStagingPath(ctx context.Context, sp string, volID string) (string, string) { - if nfs.IsNFSVolumeID(volID) { - return nfs.ToArrayVolumeID(volID), sp - } - logFields := identifiers.GetLogFields(ctx) + log := log.WithContext(ctx) if sp == "" || volID == "" { return volID, sp } stagingPath := path.Join(sp, volID) - log.WithFields(logFields).Infof("staging path is: %s", stagingPath) + log.Infof("staging path is: %s", stagingPath) return volID, path.Join(sp, volID) } func getRemnantTargetMounts(ctx context.Context, target string, fs fs.Interface) ([]gofsutil.Info, bool, error) { - logFields := identifiers.GetLogFields(ctx) + log := log.WithContext(ctx) var targetMounts []gofsutil.Info var found bool mounts, err := getMounts(ctx, fs) @@ -236,7 +233,7 @@ func getRemnantTargetMounts(ctx context.Context, target string, fs fs.Interface) for _, mount := range mounts { if strings.Contains(mount.Path, target) { targetMounts = append(targetMounts, mount) - log.WithFields(logFields).Infof("matching remnantTargetMount %s target %s", target, mount.Path) + log.Infof("matching remnantTargetMount %s target %s", target, mount.Path) found = true } } @@ -244,7 +241,7 @@ func getRemnantTargetMounts(ctx context.Context, target string, fs fs.Interface) } func getTargetMount(ctx context.Context, target string, fs fs.Interface) (gofsutil.Info, bool, error) { - logFields := identifiers.GetLogFields(ctx) + log := log.WithContext(ctx) var targetMount gofsutil.Info var found bool mounts, err := getMounts(ctx, fs) @@ -256,7 +253,7 @@ func getTargetMount(ctx context.Context, target string, fs fs.Interface) (gofsut for _, mount := range mounts { if mount.Path == target { targetMount = mount - log.WithFields(logFields).Infof("matching targetMount %s target %s", + log.Infof("matching targetMount %s target %s", target, mount.Path) found = true break @@ -359,12 +356,13 @@ func getRWModeString(isRO bool) string { return "rw" } -func format(_ context.Context, source, fsType string, fs fs.Interface, opts ...string) error { - f := log.Fields{ +func format(ctx context.Context, source, fsType string, fs fs.Interface, opts ...string) error { + f := csmlog.Fields{ "source": source, "fsType": fsType, "options": opts, } + log := log.WithContext(ctx).WithFields(f) // Use 'ext4' as the default if fsType == "" { @@ -379,10 +377,10 @@ func format(_ context.Context, source, fsType string, fs fs.Interface, opts ...s } mkfsArgs = append(mkfsArgs, opts...) - log.WithFields(f).Infof("formatting with command: %s %v", mkfsCmd, mkfsArgs) + log.Infof("formatting with command: %s %v", mkfsCmd, mkfsArgs) out, err := fs.ExecCommand(mkfsCmd, mkfsArgs...) if err != nil { - log.WithFields(f).WithError(err).Errorf("formatting disk failed, output: %q", string(out)) + log.Errorf("formatting disk failed with error: %s, output: %q", err.Error(), string(out)) return errors.New(string(out)) } diff --git a/pkg/node/ephemeral.go b/pkg/node/ephemeral.go index 35cbd874..cb0a3672 100644 --- a/pkg/node/ephemeral.go +++ b/pkg/node/ephemeral.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,7 +27,6 @@ import ( "strconv" "github.com/container-storage-interface/spec/lib/go/csi" - log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -54,6 +53,7 @@ func (s *Service) ephemeralNodePublish( req *csi.NodePublishVolumeRequest) ( *csi.NodePublishVolumeResponse, error, ) { + log := log.WithContext(ctx) if _, err := s.Fs.Stat(ephemeralStagingMountPath); os.IsNotExist(err) { log.Debug("path does not exists") err = s.Fs.MkdirAll(ephemeralStagingMountPath, 0o750) @@ -162,6 +162,7 @@ func (s *Service) ephemeralNodeUnpublish( ctx context.Context, req *csi.NodeUnpublishVolumeRequest, ) error { + log := log.WithContext(ctx) volID := req.GetVolumeId() if volID == "" { return status.Error(codes.InvalidArgument, "volume ID is required") @@ -179,7 +180,7 @@ func (s *Service) ephemeralNodeUnpublish( StagingTargetPath: stagingPath, }) if err != nil { - log.Info(err) + log.Info(err.Error()) return status.Error(codes.Internal, "Inline ephemeral node unstage unpublish failed") } log.Info("Calling unpublish") diff --git a/pkg/node/node.go b/pkg/node/node.go index a0004c08..a3b98f58 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,41 +23,52 @@ import ( "context" "errors" "fmt" + "maps" "net/url" "os" "path" "path/filepath" "regexp" + "slices" "strconv" "strings" + "time" - "github.com/dell/csm-sharednfs/nfs" "github.com/dell/gonvme" "github.com/dell/gopowerstore/api" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/pkg/array" "github.com/dell/csi-powerstore/v2/pkg/controller" "github.com/dell/csi-powerstore/v2/pkg/helpers" "github.com/dell/csi-powerstore/v2/pkg/identifiers" "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" "github.com/dell/csi-powerstore/v2/pkg/identifiers/k8sutils" + "github.com/dell/csmlog" "github.com/dell/gobrick" csictx "github.com/dell/gocsi/context" "github.com/dell/gofsutil" "github.com/dell/goiscsi" "github.com/dell/gopowerstore" - log "github.com/sirupsen/logrus" + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/protobuf/proto" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" + corev1 "k8s.io/api/core/v1" + + "k8s.io/apimachinery/pkg/labels" + "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" ) -//go:generate mockgen -destination=../../mocks/NodeInterface.go -package=mocks github.com/dell/csi-powerstore/v2/pkg/node Interface -type Interface interface { - csi.NodeServer - array.Consumer -} +// Instantiate csmlog on a package level +var log = csmlog.GetLogger() + +// For unit testing +var ( + createOrUpdateJournalEntryFunc = array.CreateOrUpdateJournalEntry + isNodeConnectedToArrayFunc = isNodeConnectedToArray + checkMetroStateFunc = array.CheckMetroState +) // Opts defines service configuration options. type Opts struct { @@ -93,7 +104,6 @@ type Service struct { useNVME map[string]bool useNFS bool initialized bool - reusedHost bool isHealthMonitorEnabled bool isPodmonEnabled bool @@ -108,11 +118,17 @@ const ( // Will init ISCSIConnector, FcConnector and ControllerService if they are nil. func (s *Service) Init() error { ctx := context.Background() + log := log.WithContext(ctx) s.opts = getNodeOptions() + _, err := k8sutils.CreateKubeClientSet(s.opts.KubeConfigPath) + if err != nil { + return fmt.Errorf("failed to create Kubernetes client: %s", err.Error()) + } + s.initConnectors() - err := s.updateNodeID() + err = s.updateNodeID() if err != nil { return fmt.Errorf("can't update node id: %s", err.Error()) } @@ -137,7 +153,7 @@ func (s *Service) Init() error { } if len(nvmeInitiators) != 0 { - err = k8sutils.AddNVMeLabels(ctx, s.opts.KubeConfigPath, s.opts.KubeNodeName, "hostnqn-uuid", nvmeInitiators) + err = k8sutils.Kubeclient.AddNVMeLabels(ctx, s.opts.KubeNodeName, "hostnqn-uuid", nvmeInitiators) if err != nil { log.Warnf("Unable to add hostnqn uuid label for node %s: %v", s.opts.KubeNodeName, err.Error()) } @@ -216,7 +232,6 @@ func (s *Service) Init() error { func (s *Service) initConnectors() { gobrick.SetLogger(&identifiers.CustomLogger{}) - if s.iscsiConnector == nil { s.iscsiConnector = gobrick.NewISCSIConnector( gobrick.ISCSIConnectorParams{ @@ -261,11 +276,10 @@ func (s *Service) initConnectors() { // Check for duplicate hostnqn uuids func (s *Service) checkForDuplicateUUIDs() { - nodeUUIDs := make(map[string]string) duplicateUUIDs := make(map[string]string) var err error - nodeUUIDs, err = k8sutils.GetNVMeUUIDs(context.Background(), s.opts.KubeConfigPath) + nodeUUIDs, err := k8sutils.Kubeclient.GetNVMeUUIDs(context.Background()) if err != nil { log.Errorf("Unable to check uuids") return @@ -283,8 +297,8 @@ func (s *Service) checkForDuplicateUUIDs() { // NodeStageVolume prepares volume to be consumed by node publish by connecting volume to the node func (s *Service) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { - logFields := identifiers.GetLogFields(ctx) - + log := log.WithContext(ctx) + logFields := csmlog.ExtractFieldsFromContext(ctx) if req.GetVolumeCapability() == nil { return nil, status.Error(codes.InvalidArgument, "volume capability is required") } @@ -307,6 +321,8 @@ func (s *Service) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeR arrayID := volumeHandle.LocalArrayGlobalID protocol := volumeHandle.Protocol remoteVolumeID := volumeHandle.RemoteUUID + remoteArrayID := volumeHandle.RemoteArrayGlobalID + _, stagingPath := getStagingPath(ctx, req.GetStagingTargetPath(), id) arr, ok := s.Arrays()[arrayID] if !ok { @@ -315,6 +331,33 @@ func (s *Service) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeR client := arr.GetClient() + var remoteArray *array.PowerStoreArray + isMetroFractured := false + metroSession := &array.MetroFracturedResponse{ + IsFractured: false, + } + localVolumeDemoted := false + + if volumeHandle.IsMetro() { + remoteArray, ok = s.Arrays()[remoteArrayID] + if !ok { + return nil, status.Errorf(codes.InvalidArgument, "failed to find remote array with ID %s", remoteArrayID) + } + + metroSession, localVolumeDemoted, err = checkMetroStateFunc(ctx, volumeHandle, arr.GetClient(), remoteArray.GetClient()) + if err != nil { + return nil, err + } + + isMetroFractured = metroSession.IsFractured + if isMetroFractured { + log.Warnf("[METRO] metro volume %s is in a fractured state", req.GetVolumeId()) + } + if localVolumeDemoted { + log.Warnf("[METRO] metro volume %s has been demoted", req.GetVolumeId()) + } + } + var stager VolumeStager if protocol == "nfs" { stager = &NFSStager{ @@ -330,27 +373,109 @@ func (s *Service) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeR } } - response, err := stager.Stage(ctx, req, logFields, s.Fs, id, false, client) - if err != nil { - return nil, err + localStaged := false + remoteStaged := false + + var response *csi.NodeStageVolumeResponse + + // For NFS , no need to check for connectivity before attempting staging. + if protocol == "nfs" { + response, err = stager.Stage(ctx, req, stagingPath, s.nodeID, logFields, s.Fs, id, false, client) + if err != nil { + return nil, err + } + return response, nil } - if remoteVolumeID != "" { // For Remote Metro volume - log.Info("Staging remote metro volume") - // need to change the staging path for nfs - if nfs.IsNFSVolumeID(req.VolumeId) { - req.StagingTargetPath = nfs.NfsExportDirectory + // For block volumes, stage only if array has connectivity to this node. + // This supports non-uniform metro configuration and will support Multi-az for powerstore non-metro volumes (future) + nodeConnectedToLocalArray := isNodeConnectedToArrayFunc(ctx, s.nodeID, arr) + if nodeConnectedToLocalArray { + resp, err := stager.Stage(ctx, req, stagingPath, s.nodeID, logFields, s.Fs, id, false, client) + if err != nil { + if isMetroFractured && localVolumeDemoted { + // expected failure if Metro is Fractured and local array is down + log.Infof("[METRO] Could not stage volume %s on node %s for array %s due to Metro Session Fracture", id, s.opts.KubeNodeName, arr.Endpoint) + } else { + log.Errorf("Failed to stage volume %s for array %s: %s", id, arr.Endpoint, err) + return nil, err + } + } else { + log.Infof("Staged volume %s for array %s", id, arr.Endpoint) + localStaged = true + response = resp + } + } else { + log.Warnf("local volume %s has no connectivity to node %s. skipping staging.", id, s.opts.KubeNodeName) + } + + nodeConnectedToRemoteArray := false + if volumeHandle.IsMetro() { // For Remote Metro volume + nodeConnectedToRemoteArray = isNodeConnectedToArrayFunc(ctx, s.nodeID, remoteArray) + if nodeConnectedToRemoteArray { + log.Infof("Staging remote metro volume %s for volume %s", remoteVolumeID, id) + resp, err := stager.Stage(ctx, req, stagingPath, s.nodeID, logFields, s.Fs, remoteVolumeID, true, remoteArray.GetClient()) + if err != nil { + if isMetroFractured && !localVolumeDemoted { + // expected failure if Metro is Fractured and remote array is down + log.Infof("[METRO] Could not stage volume %s on node %s for array %s due to Metro Session Fracture", id, s.opts.KubeNodeName, remoteArray.Endpoint) + } else { + log.Errorf("Failed to stage volume %s for array %s: %s", id, remoteArray.Endpoint, err) + return nil, err + } + } else { + log.Infof("Remote volume %s staged", remoteVolumeID) + remoteStaged = true + response = resp + } + } else { + log.Debugf("skipping staging remote metro %s, node has not been registered with the remote array %s", remoteVolumeID, remoteArrayID) } - response, err = stager.Stage(ctx, req, logFields, s.Fs, remoteVolumeID, true, client) } - return response, err + // at least one stage should succeed for non-metro, non-uniform metro, and uniform metro + // if a staging fails for uniform metro, the failed request will be deferred by adding to the volume journal + if !localStaged && !remoteStaged { + return nil, status.Error(codes.Internal, "failed to stage volume") + } + + if volumeHandle.IsMetro() && nodeConnectedToLocalArray && nodeConnectedToRemoteArray { + if (localStaged && !remoteStaged) || (!localStaged && remoteStaged) { + deferredRequest, err := proto.Marshal(req) + if err != nil { + log.Errorf("[METRO] Error marshalling req: %s", err.Error()) + return nil, err + } + + deferredArrayID := arrayID + if !remoteStaged { + deferredArrayID = remoteArrayID + } + + err = createOrUpdateJournalEntryFunc(ctx, metroSession.VolumeName, volumeHandle, deferredArrayID, s.opts.KubeNodeName, "NodeStageVolume", deferredRequest) + if err != nil { + log.Errorf("Could not create journal entry for operation %s for volume %s node %s array %s", "NodeStageVolume", id, s.opts.KubeNodeName, arrayID) + return nil, err + } + + log.Infof("[METRO] Metro volume %s created journal entry for operation %s for volume %s node %s array %s", id, "NodeStageVolume", id, s.opts.KubeNodeName, arrayID) + } + } + return response, nil } // NodeUnstageVolume reverses steps done in NodeStage by disconnecting volume from the node func (s *Service) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { - logFields := identifiers.GetLogFields(ctx) + log := log.WithContext(ctx) var err error + var reqID string + logFields := csmlog.ExtractFieldsFromContext(ctx) + headers, ok := metadata.FromIncomingContext(ctx) + if ok { + if req, ok := headers["csi.requestid"]; ok && len(req) > 0 && req[0] != "" { + reqID = req[0] + } + } id := req.GetVolumeId() if id == "" { @@ -384,6 +509,18 @@ func (s *Service) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVol id, stagingPath = getStagingPath(ctx, stagingPath, id) + vol, err := arr.Client.GetVolume(ctx, id) + if err != nil { + if apiError, ok := err.(gopowerstore.APIError); ok { + if !apiError.NotFound() { + return nil, status.Errorf(codes.Internal, "issue getting volume %s, error %s", id, err.Error()) + } + + // Not found due to potentially deleted volume through UI. Still need to unstage. + log.Infof("Volume with ID %s not found", id) + } + } + device, err := unstageVolume(ctx, stagingPath, id, logFields, err, s.Fs) if err != nil { return nil, err @@ -391,10 +528,7 @@ func (s *Service) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVol if remoteVolumeID != "" { // For Remote Metro volume log.Info("Unstaging remote metro volume") _, remoteStagingPath := getStagingPath(ctx, req.GetStagingTargetPath(), remoteVolumeID) - // need to change the staging path for nfs - if nfs.IsNFSVolumeID(req.VolumeId) { - _, remoteStagingPath = getStagingPath(ctx, nfs.NfsExportDirectory, remoteVolumeID) - } + _, err = unstageVolume(ctx, remoteStagingPath, remoteVolumeID, logFields, err, s.Fs) if err != nil { return nil, err @@ -408,47 +542,129 @@ func (s *Service) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVol if device != "" { err := createMapping(id, device, s.opts.TmpDir, s.Fs) if err != nil { - log.WithFields(logFields).Warningf("failed to create vol to device mapping: %s", err.Error()) + log.Warnf("failed to create vol to device mapping : %s", err.Error()) } } else { device, err = getMapping(id, s.opts.TmpDir, s.Fs) if err != nil { - log.WithFields(logFields).Info("no device found. skip device removal") + log.Info("no device found. skip device removal") return &csi.NodeUnstageVolumeResponse{}, nil } } - f := log.Fields{"Device": device} + f := csmlog.Fields{"Device": device} - connectorCtx := identifiers.SetLogFields(context.Background(), logFields) + connectorCtx := csmlog.SetLogFields(context.Background(), logFields) if s.useNVME[arr.GlobalID] { err = s.nvmeConnector.DisconnectVolumeByDeviceName(connectorCtx, device) } else if s.useFC[arr.GlobalID] { - err = s.fcConnector.DisconnectVolumeByDeviceName(connectorCtx, device) + log.Infof("WWN of Volume for unstaging: %s", vol.Wwn) + + volumeWWN := vol.Wwn + err = s.disconnectFCVolume(ctx, reqID, id, arrayID, device, strings.Split(volumeWWN, ".")[1], logFields) } else { err = s.iscsiConnector.DisconnectVolumeByDeviceName(connectorCtx, device) } if err != nil { - log.WithFields(logFields).Error(err) + log.WithFields(logFields).Errorf("failed to disconnect volume: %s", err.Error()) return nil, err } - log.WithFields(logFields).WithFields(f).Info("block device removal complete") + + log.WithFields(logFields).WithFields(f).Infof("block device %s removal completed :", device) err = deleteMapping(id, s.opts.TmpDir, s.Fs) if err != nil { - log.WithFields(logFields).Warningf("failed to remove vol to Dev mapping: %s", err.Error()) + log.WithFields(logFields).Warnf("failed to delete vol to device mapping : %s", err.Error()) } return &csi.NodeUnstageVolumeResponse{}, nil } -func unstageVolume(ctx context.Context, stagingPath, id string, logFields log.Fields, err error, fs fs.Interface) (string, error) { +// New method implementing FC volume disconnection with retry logic similar to PowerMax +func (s *Service) disconnectFCVolume(ctx context.Context, reqID, volumeID, arrayID, device, volumeWWN string, logFields map[string]interface{}) error { + log := log.WithContext(ctx) + var err error + maxDisconnectRetries := identifiers.GetVolumeDisconnectMaxRetries() + timeout := identifiers.GetVolumeDisconnectTimeout() + retryInterval := identifiers.GetVolumeDisconnectRetryInterval() + + f := csmlog.Fields{ + "CSIRequestID": reqID, + "VolumeID": volumeID, + "ArrayID": arrayID, + "Device": device, + "WWN": volumeWWN, + } + log.Infof("WWN of Volume for disconnectingFCVolume: %s", volumeWWN) + + for i := 1; i <= maxDisconnectRetries; i++ { + f["Retry"] = i + log.WithFields(f).Info("NodeUnstageVolume disconnect volume FC") + + // Create context with timeout for disconnection + disconnectCtx, cancel := context.WithTimeout(ctx, timeout) + disconnectCtx = csmlog.SetLogFields(disconnectCtx, logFields) + + if volumeWWN != "" { + // Preferred: Use WWN-based disconnection (more reliable) + err = s.fcConnector.DisconnectVolumeByWWN(disconnectCtx, volumeWWN) + } else { + // Fallback: Use device name based disconnection + err = s.fcConnector.DisconnectVolumeByDeviceName(disconnectCtx, device) + } + + cancel() + + if err == nil { + log.WithFields(f).Debug("FC disconnect volume complete") + + // Clean up symlink if WWN was available + if volumeWWN != "" { + symlinkPath, _, err := gofsutil.WWNToDevicePathX(ctx, volumeWWN) + if err != nil { + log.WithFields(f).Warnf("failed to resolve symlink path for WWN %s: %s", volumeWWN, err) + } else if symlinkPath != "" { + if removeErr := os.Remove(symlinkPath); removeErr != nil && !os.IsNotExist(removeErr) { + log.WithFields(f).Warnf("failed to remove symlink at path %s: %s", symlinkPath, removeErr.Error()) + } + } + } + return nil + } + + log.WithFields(f).Errorf("error disconnecting volume for retry %d: %s", i, err.Error()) + + if i < maxDisconnectRetries { + time.Sleep(retryInterval) + + // Additional check: verify if device still exists before retrying + if volumeWWN != "" { + devPath, err := gofsutil.WWNToDevicePath(ctx, volumeWWN) + if err != nil { + log.WithFields(f).Warnf("failed to resolve device path for WWN %s: %v", volumeWWN, err) + return nil + } + if devPath == "" { + log.WithFields(f).Info("device no longer exists, considering disconnect successful") + return nil + } + } + } + } + + return status.Errorf(codes.Internal, + "FC disconnectVolume exceeded retry limit %d for volume %s, device %s, WWN %s", + maxDisconnectRetries, volumeID, device, volumeWWN) +} + +func unstageVolume(ctx context.Context, stagingPath, id string, logFields csmlog.Fields, err error, fs fs.Interface) (string, error) { logFields["ID"] = id logFields["StagingPath"] = stagingPath - ctx = identifiers.SetLogFields(ctx, logFields) + ctx = csmlog.SetLogFields(ctx, logFields) + log := log.WithContext(ctx).WithFields(logFields) - log.WithFields(logFields).Info("calling unstage") + log.Info("calling unstage") device, err := getStagedDev(ctx, stagingPath, fs) if err != nil { @@ -458,16 +674,16 @@ func unstageVolume(ctx context.Context, stagingPath, id string, logFields log.Fi if device != "" { _, device = path.Split(device) - log.WithFields(logFields).Infof("active mount exist") + log.Info("active mount exist") err = fs.GetUtil().Unmount(ctx, stagingPath) if err != nil { return "", status.Errorf(codes.Internal, "could not unmount dev %s: %s", device, err.Error()) } - log.WithFields(logFields).Infof("unmount without error") + log.Info("unmount without error") } else { // no mounts - log.WithFields(logFields).Infof("no mounts found") + log.Info("no active mounts found") } err = fs.Remove(stagingPath) @@ -483,18 +699,22 @@ func unstageVolume(ctx context.Context, stagingPath, id string, logFields log.Fi return "", status.Errorf(codes.Internal, "failed to delete mount path %s: %s", stagingPath, err.Error()) } - log.WithFields(logFields).Infof("target mount file deleted") + log.Info("target mount file deleted") return device, nil } -func removeRemnantMounts(ctx context.Context, stagingPath string, fs fs.Interface, logFields log.Fields) (string, error) { - log.WithFields(logFields).Infof("getting remnant mount") +func removeRemnantMounts(ctx context.Context, stagingPath string, fs fs.Interface, logFields csmlog.Fields) (string, error) { + log := log.WithContext(ctx).WithFields(logFields) + log.Info("finding remnant mount") mounts, found, err := getRemnantTargetMounts(ctx, stagingPath, fs) - if !found || err != nil { + if err != nil { return "", fmt.Errorf("could not reliably determine remnant mounts for path %s: %s", stagingPath, err.Error()) } + if !found { + return "", fmt.Errorf("no remnant mounts for %s", stagingPath) + } - log.WithFields(logFields).Infof("%d remnant mount exist", len(mounts)) + log.Infof("%d remnant mount exist", len(mounts)) for _, mount := range mounts { delete(logFields, "StagingPath") logFields["RemnantPath"] = mount.Path @@ -502,7 +722,7 @@ func removeRemnantMounts(ctx context.Context, stagingPath string, fs fs.Interfac if err != nil { return "", fmt.Errorf("could not unmount dev %s: %s", mount.Path, err.Error()) } - log.WithFields(logFields).Infof("unmount without error") + log.Info("unmount without error") } delete(logFields, "RemnantPath") @@ -515,12 +735,10 @@ func removeRemnantMounts(ctx context.Context, stagingPath string, fs fs.Interfac // NodePublishVolume publishes volume to the node by mounting it to the target path func (s *Service) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { - logFields := identifiers.GetLogFields(ctx) + log := log.WithContext(ctx) + logFields := csmlog.ExtractFieldsFromContext(ctx) var ephemeralVolume bool - isHBN := nfs.IsNFSVolumeID(req.VolumeId) - if isHBN { - req.VolumeId = nfs.ToArrayVolumeID(req.VolumeId) - } + ephemeral, ok := req.VolumeContext["csi.storage.k8s.io/ephemeral"] if ok { ephemeralVolume = strings.ToLower(ephemeral) == "true" @@ -552,12 +770,7 @@ func (s *Service) NodePublishVolume(ctx context.Context, req *csi.NodePublishVol id = volumeHandle.LocalUUID protocol := volumeHandle.Protocol - stagingPath := req.GetStagingTargetPath() - - if !isHBN { - // append additional path to be able to do bind mounts - _, stagingPath = getStagingPath(ctx, req.GetStagingTargetPath(), id) - } + id, stagingPath := getStagingPath(ctx, req.GetStagingTargetPath(), id) isRO := req.GetReadonly() volumeCapability := req.GetVolumeCapability() @@ -566,9 +779,9 @@ func (s *Service) NodePublishVolume(ctx context.Context, req *csi.NodePublishVol logFields["TargetPath"] = targetPath logFields["StagingPath"] = stagingPath logFields["ReadOnly"] = req.GetReadonly() - ctx = identifiers.SetLogFields(ctx, logFields) + ctx = csmlog.SetLogFields(ctx, logFields) - log.WithFields(logFields).Info("calling publish") + log.WithFields(logFields).Info("calling node publish volume") var publisher VolumePublisher @@ -590,7 +803,8 @@ func (s *Service) NodePublishVolume(ctx context.Context, req *csi.NodePublishVol // NodeUnpublishVolume unpublishes volume from the node by unmounting it from the target path func (s *Service) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { - logFields := identifiers.GetLogFields(ctx) + logFields := csmlog.ExtractFieldsFromContext(ctx) + log := log.WithFields(logFields) var err error targetPath := req.GetTargetPath() @@ -611,8 +825,8 @@ func (s *Service) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublis } logFields["ID"] = volID logFields["TargetPath"] = targetPath - ctx = identifiers.SetLogFields(ctx, logFields) - log.WithFields(logFields).Info("calling unpublish") + ctx = csmlog.SetLogFields(ctx, logFields) + log.Info("calling unpublish") _, found, err := getTargetMount(ctx, targetPath, s.Fs) if err != nil { @@ -623,11 +837,11 @@ func (s *Service) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublis if !found { // no mounts - log.WithFields(logFields).Infof("no mounts found") + log.Info("no mounts found") return &csi.NodeUnpublishVolumeResponse{}, nil } - log.WithFields(logFields).Infof("active mount exist") + log.Info("active mount exist") err = s.Fs.GetUtil().Unmount(ctx, targetPath) if err != nil { return nil, status.Errorf(codes.Internal, @@ -641,7 +855,7 @@ func (s *Service) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublis return nil, status.Errorf(codes.Internal, "Failed to remove target path: %s as part of NodeUnpublish: %s", targetPath, err.Error()) } - log.WithFields(logFields).Info("unpublish complete") + log.Info("unpublish complete") log.Debug("Checking for ephemeral after node unpublish") if ephemeralVolume { @@ -905,6 +1119,7 @@ func (s *Service) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolume // NodeExpandVolume expands the volume by re-scanning and resizes filesystem if needed func (s *Service) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) { + log := log.WithContext(ctx) var reqID string var err error headers, ok := metadata.FromIncomingContext(ctx) @@ -914,11 +1129,6 @@ func (s *Service) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolum } } - isHBN := nfs.IsNFSVolumeID(req.VolumeId) - if isHBN { - req.VolumeId = nfs.ToArrayVolumeID(req.VolumeId) - } - // Get the VolumeID and validate against the volume volumeHandle, err := array.ParseVolumeID(ctx, req.VolumeId, s.DefaultArray(), nil) if err != nil { @@ -964,18 +1174,14 @@ func (s *Service) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolum } } - log.Debug("Volume name: ", vol.Name) + log.Debugf("Volume name: %s", vol.Name) volumeWWN := vol.Wwn // Locate and fetch all (multipath/regular) mounted paths using this volume var devMnt *gofsutil.DeviceMountInfo var targetmount string - if isHBN { - devMnt, err = s.Fs.GetUtil().GetMountInfoFromDevice(ctx, id) - } else { - devMnt, err = s.Fs.GetUtil().GetMountInfoFromDevice(ctx, vol.Name) - } + devMnt, err = s.Fs.GetUtil().GetMountInfoFromDevice(ctx, vol.Name) // Stop block volume expansion if metro session is paused // User needs to resume it first. @@ -1050,7 +1256,7 @@ func (s *Service) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolum size := req.GetCapacityRange().GetRequiredBytes() - f := log.Fields{ + f := csmlog.Fields{ "CSIRequestID": reqID, "VolumeName": vol.Name, "VolumePath": targetPath, @@ -1058,7 +1264,6 @@ func (s *Service) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolum "VolumeWWN": volumeWWN, } log.WithFields(f).Info("Calling resize the file system") - if !s.useNVME[arr.GlobalID] { // Rescan the device for the volume expanded on the array for _, device := range devMnt.DeviceNames { @@ -1145,6 +1350,7 @@ func (s *Service) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolum } func (s *Service) nodeExpandRawBlockVolume(ctx context.Context, volumeWWN string) (*csi.NodeExpandVolumeResponse, error) { + log := log.WithContext(ctx) log.Info(" Block volume expansion. Will try to perform a rescan...") wwnNum := strings.Replace(volumeWWN, "naa.", "", 1) deviceNames, err := s.Fs.GetUtil().GetSysBlockDevicesForVolumeWWN(context.Background(), wwnNum) @@ -1241,6 +1447,7 @@ func (s *Service) NodeGetCapabilities(_ context.Context, _ *csi.NodeGetCapabilit func (s *Service) NodeGetInfo(ctx context.Context, _ *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { // Create the topology keys // /-: true + log := log.WithContext(ctx) resp := &csi.NodeGetInfoResponse{ NodeId: s.nodeID, AccessibleTopology: &csi.Topology{ @@ -1248,11 +1455,16 @@ func (s *Service) NodeGetInfo(ctx context.Context, _ *csi.NodeGetInfoRequest) (* }, } + nodeLabels, err := k8sutils.Kubeclient.GetNodeLabels(ctx, s.opts.KubeNodeName) + if err != nil { + log.Warnf("failed to get Node Labels with error: %s", err.Error()) + } + for _, arr := range s.Arrays() { if isNFSEnabled, err := identifiers.IsNFSServiceEnabled(ctx, arr.GetClient()); err != nil { log.Errorf("failed to validate NFS service for the array: %s", err.Error()) } else if isNFSEnabled { - log.Info("NFS service is enabled on the array ", arr.GetGlobalID()) + log.Infof("NFS service is enabled on the array %s ", arr.GetGlobalID()) // we will chop off port from the host if present. port, err := ExtractPort(arr.Endpoint) _, err = getOutboundIP(arr.GetIP(), port, s.Fs) @@ -1300,21 +1512,35 @@ func (s *Service) NodeGetInfo(ctx context.Context, _ *csi.NodeGetInfoRequest) (* log.Errorf("couldn't get targets from array: %s", err.Error()) continue } + var nvmeTargets []gonvme.NVMeTarget + networkIDs := map[string]struct{}{} for _, address := range infoList { + // discovering with one portal returns all targets in the network + // so if we already discovered an address with this network ID, continue + if _, ok := networkIDs[address.NetworkID]; ok { + continue + } + + // discover the target // doesn't matter how many portals are present, discovering from any one will list out all targets - nvmeIP := strings.Split(address.Portal, ":") - log.Info("Trying to discover NVMe target from portal ", nvmeIP[0]) - nvmeTargets, err = s.nvmeLib.DiscoverNVMeTCPTargets(nvmeIP[0], false) + nvmeIP := strings.Split(address.Portal, ":")[0] + log.Infof("Trying to discover NVMe targets from portal %s on network %s", nvmeIP, address.NetworkID) + discoveredTargets, err := s.nvmeLib.DiscoverNVMeTCPTargets(nvmeIP, false) if err != nil { - log.Error("couldn't discover targets") + log.Errorf("discovering portal: %s: %v", nvmeIP, err) continue } - break + + nvmeTargets = append(nvmeTargets, discoveredTargets...) + + // mark this network ID as discovered so we don't discover another portal in the same network + // since it will return all the same target information already seen + networkIDs[address.NetworkID] = struct{}{} } loginToAtleastOneTarget := false for _, target := range nvmeTargets { - log.Info("Logging to NVMe target ", target) + log.Infof("Logging to NVMe target %v", target) err = s.nvmeLib.NVMeTCPConnect(target, false) if err != nil { log.Errorf("couldn't connect to the nvme target") @@ -1332,21 +1558,10 @@ func (s *Service) NodeGetInfo(ctx context.Context, _ *csi.NodeGetInfoRequest) (* } } else if s.useFC[arr.GlobalID] { // Check node initiators connection to array - nodeID := s.nodeID - if s.reusedHost { - ipList := identifiers.GetIPListFromString(nodeID) - if ipList == nil || len(ipList) == 0 { - log.Errorf("can't find ip in nodeID %s", nodeID) - continue - } - ip := ipList[len(ipList)-1] - nodeID = nodeID[:len(nodeID)-len(ip)-1] - } - - host, err := arr.GetClient().GetHostByName(ctx, nodeID) + host, err := arr.GetClient().GetHostByName(ctx, s.nodeID) if err != nil { - log.WithFields(log.Fields{ - "hostName": nodeID, + log.WithFields(csmlog.Fields{ + "hostName": s.nodeID, "error": err, }).Error("could not find host on PowerStore array") continue @@ -1357,10 +1572,11 @@ func (s *Service) NodeGetInfo(ctx context.Context, _ *csi.NodeGetInfoRequest) (* continue } - if len(host.Initiators[0].ActiveSessions) != 0 { + fcInitiatorsWithActiveSessionCount := countActiveSessionsInitiators(host) + if fcInitiatorsWithActiveSessionCount > 0 { resp.AccessibleTopology.Segments[identifiers.Name+"/"+arr.GetIP()+"-fc"] = "true" } else { - log.WithFields(log.Fields{ + log.WithFields(csmlog.Fields{ "hostName": host.Name, "initiator": host.Initiators[0].PortName, }).Error("there is no active FC sessions") @@ -1374,25 +1590,37 @@ func (s *Service) NodeGetInfo(ctx context.Context, _ *csi.NodeGetInfoRequest) (* } var ipAddress string var iscsiTargets []goiscsi.ISCSITarget + networkIDs := map[string]struct{}{} for _, address := range infoList { + // discovering with one portal returns all targets in the network + // so if we already discovered an address with this network ID, continue + if _, ok := networkIDs[address.NetworkID]; ok { + continue + } + // first check if this portal is reachable from this machine or not if ReachableEndPoint(address.Portal) { ipAddressList := splitIPAddress(address.Portal) ipAddress = ipAddressList[0] // doesn't matter how many portals are present, discovering from any one will list out all targets - log.Info("Trying to discover iSCSI target from portal ", ipAddress) + log.Infof("Trying to discover iSCSI target from portal %s", ipAddress) ipInterface, err := s.iscsiLib.GetInterfaceForTargetIP(ipAddress) if err != nil { log.Errorf("couldn't get interface: %s", err.Error()) continue } - iscsiTargets, err = s.iscsiLib.DiscoverTargetsWithInterface(address.Portal, ipInterface[ipAddress], false) + discoveredTargets, err := s.iscsiLib.DiscoverTargetsWithInterface(address.Portal, ipInterface[ipAddress], false) if err != nil { - log.Error("couldn't discover targets") + log.Errorf("couldn't discover targets: %s", err.Error()) continue } - break + + iscsiTargets = append(iscsiTargets, discoveredTargets...) + + // mark this network ID as discovered so we don't discover another portal in the same network + // since it will return all the same target information already seen + networkIDs[address.NetworkID] = struct{}{} } log.Debugf("Portal is not rechable from the node") } @@ -1403,7 +1631,7 @@ func (s *Service) NodeGetInfo(ctx context.Context, _ *csi.NodeGetInfoRequest) (* loginToAtleastOneTarget := false for _, target := range iscsiTargets { if ReachableEndPoint(target.Portal) { - log.Info("Logging to Iscsi target ", target) + log.Infof("Logging to Iscsi target %v", target) if s.opts.EnableCHAP { log.Debug("Setting CHAP Credentials before login") err = s.iscsiLib.SetCHAPCredentials(target, s.opts.CHAPUsername, s.opts.CHAPPassword) @@ -1429,6 +1657,8 @@ func (s *Service) NodeGetInfo(ctx context.Context, _ *csi.NodeGetInfoRequest) (* } } } + + updateMetroToplogy(arr, nodeLabels, resp) } var maxVolumesPerNode int64 @@ -1439,11 +1669,8 @@ func (s *Service) NodeGetInfo(ctx context.Context, _ *csi.NodeGetInfoRequest) (* } // Check for node label 'max-powerstore-volumes-per-node'. If present set 'maxVolumesPerNode' to this value. - labels, err := k8sutils.GetNodeLabels(ctx, s.opts.KubeConfigPath, s.opts.KubeNodeName) - if err != nil { - log.Warnf("failed to get Node Labels with error: %s", err.Error()) - } else if labels != nil { - if val, ok := labels[maxPowerstoreVolumesPerNodeLabel]; ok { + if nodeLabels != nil { + if val, ok := nodeLabels[maxPowerstoreVolumesPerNodeLabel]; ok { maxVols, err := strconv.ParseInt(val, 10, 64) if err != nil { log.Warnf("invalid value '%s' specified for 'max-powerstore-volumes-per-node' node label", val) @@ -1463,11 +1690,22 @@ func (s *Service) NodeGetInfo(ctx context.Context, _ *csi.NodeGetInfoRequest) (* return resp, nil } +// Count the FC initiators with active sessions +func countActiveSessionsInitiators(host gopowerstore.Host) int { + fcInitiatorsWithActiveSessionCount := 0 + for _, initiator := range host.Initiators { + if len(initiator.ActiveSessions) != 0 { + fcInitiatorsWithActiveSessionCount++ + } + } + return fcInitiatorsWithActiveSessionCount +} + func (s *Service) updateNodeID() error { if s.nodeID == "" { hostID, err := s.Fs.ReadFile(s.opts.NodeIDFilePath) if err != nil { - log.WithFields(log.Fields{ + log.WithFields(csmlog.Fields{ "path": s.opts.NodeIDFilePath, "error": err, }).Error("Could not read Node ID file") @@ -1482,7 +1720,7 @@ func (s *Service) updateNodeID() error { // we will chop off port from the host if present. port, err := ExtractPort(defaultArray.Endpoint) ip, err := getOutboundIP(defaultArray.GetIP(), port, s.Fs) - log.Debug("Outbound IP address: ", ip.String()) + log.Debugf("Outbound IP address: %s", ip.String()) // When Authorization v2 is enabled the host IP address will be localhost. We should get the actual IP else volume will not mount if ip.String() == "127.0.0.1" || ip.String() == "localhost" { @@ -1493,9 +1731,9 @@ func (s *Service) updateNodeID() error { } } - log.Debug("Outbound IP address after check: ", ip.String()) + log.Debugf("Outbound IP address after check: %s", ip.String()) if err != nil { - log.WithFields(log.Fields{ + log.WithFields(csmlog.Fields{ "endpoint": s.DefaultArray().GetIP(), "error": err, }).Error("Could not connect to PowerStore array") @@ -1508,7 +1746,7 @@ func (s *Service) updateNodeID() error { if len(nodeID) > powerStoreMaxNodeNameLength { err := errors.New("node name prefix is too long") - log.WithFields(log.Fields{ + log.WithFields(csmlog.Fields{ "value": s.opts.NodeNamePrefix, "error": err, }).Error("Invalid Node ID") @@ -1520,11 +1758,10 @@ func (s *Service) updateNodeID() error { } func (s *Service) getInitiators() ([]string, []string, []string, error) { - ctx := context.Background() - var iscsiAvailable bool var fcAvailable bool var nvmeAvailable bool + ctx := context.Background() iscsiInitiators, err := s.iscsiConnector.GetInitiatorName(ctx) if err != nil { @@ -1567,6 +1804,7 @@ func (s *Service) getInitiators() ([]string, []string, []string, error) { func (s *Service) getNodeFCPorts(ctx context.Context) ([]string, error) { var err error var initiators []string + log := log.WithContext(ctx) defer func() { initiators := initiators @@ -1692,7 +1930,6 @@ func (s *Service) setupHost(initiators []string, client gopowerstore.Client, arr return fmt.Errorf("failed to update host name: %v", err) } } - s.reusedHost = true } s.initialized = true @@ -1700,13 +1937,14 @@ func (s *Service) setupHost(initiators []string, client gopowerstore.Client, arr } func (s *Service) modifyHostName(ctx context.Context, client gopowerstore.Client, nodeID string, hostID string) error { + log := log.WithContext(ctx) modifyParams := gopowerstore.HostModify{} modifyParams.Name = &nodeID _, err := client.ModifyHost(ctx, &modifyParams, hostID) if err != nil { return err } - log.Info("Updated nodeID ", nodeID) + log.Infof("Updated nodeID %s", nodeID) return nil } @@ -1767,12 +2005,130 @@ var ( } ) +func (s *Service) createHost(ctx context.Context, initiators []string) (string, error) { + hostConnectivity := false + metroTopology := false + for _, arr := range getArrayfn(s) { + if arr.MetroTopology != "" { + if hostConnectivity { + return "", fmt.Errorf("host connectivity and metro topology cannot be set at the same time") + } + metroTopology = true + } + if arr.HostConnectivity != nil { + if metroTopology { + return "", fmt.Errorf("host connectivity and metro topology cannot be set at the same time") + } + hostConnectivity = true + } + } + if hostConnectivity { + return s.createHostHostConnectivity(ctx, initiators) + } + return s.createHostMetroTopologyAndLocal(ctx, initiators) +} + // register host -func (s *Service) createHost( +func (s *Service) createHostHostConnectivity(ctx context.Context, initiators []string) (string, error) { + log := log.WithContext(ctx) + node, err := k8sutils.Kubeclient.GetNode(context.Background(), s.opts.KubeNodeName) + if err != nil { + return "", fmt.Errorf("[createHost] Failed to get node %s: %v", s.opts.KubeNodeName, err) + } + var primaryArrayID string + + for _, arr := range getArrayfn(s) { + var conn gopowerstore.HostConnectivityEnum + log.Infof("[createHost] Processing array %s (%s)", arr.GlobalID, arr.IP) + // 1) Skip if already registered + if getIsHostAlreadyRegistered(s, ctx, arr.GetClient(), initiators) { + log.Infof("[createHost] Already registered on %s, skipping", arr.GlobalID) + if primaryArrayID == "" { + primaryArrayID = arr.GlobalID + } + continue + } + // 2) Metro vs Non‑Metro + log.Debugf("[createHost] Processing array %s (%d)(%v)", arr.GlobalID, arr.HostConnectivity.Local.Size(), &arr.HostConnectivity.Metro) + if arr.HostConnectivity.Local.Size() > 0 { + match, err := nodeMatchSelector(node, &arr.HostConnectivity.Local, conn) + if err != nil { + return "", fmt.Errorf("[createHost] Error matching host connectivity selector for array %s: %v", arr.GlobalID, err) + } + if match { + log.Infof("[createHost] Zone match on %s, registering host locally", arr.GlobalID) + conn = gopowerstore.HostConnectivityEnumLocalOnly + } + } + + // 3) Metro: match zone with array topology + match, err := nodeMatchSelector(node, &arr.HostConnectivity.Metro.ColocatedLocal, conn) + if err != nil { + return "", fmt.Errorf("[createHost] Error matching host connectivity selector for array %s: %v", arr.GlobalID, err) + } + if match { + log.Infof("[createHost] Metro & label match on %s, registering as ColocatedLocal", arr.GlobalID) + conn = gopowerstore.HostConnectivityEnumMetroOptimizeLocal + } + + match, err = nodeMatchSelector(node, &arr.HostConnectivity.Metro.ColocatedRemote, conn) + if err != nil { + return "", fmt.Errorf("[createHost] Error matching host connectivity selector for array %s: %v", arr.GlobalID, err) + } + if match { + log.Infof("[createHost] Metro & label match on %s, registering as ColocatedRemote", arr.GlobalID) + conn = gopowerstore.HostConnectivityEnumMetroOptimizeRemote + } + + match, err = nodeMatchSelector(node, &arr.HostConnectivity.Metro.ColocatedBoth, conn) + if err != nil { + return "", fmt.Errorf("[createHost] Error matching host connectivity selector for array %s: %v", arr.GlobalID, err) + } + if match { + log.Infof("[createHost] Metro & label match on %s, registering as ColocatedBoth", arr.GlobalID) + conn = gopowerstore.HostConnectivityEnumMetroOptimizeBoth + } + if conn == "" { + log.Infof("[createHost] Metro & label mismatch on %s, skip registration for this host", arr.GlobalID) + continue + } + if err := registerHostFunc(s, ctx, arr.GetClient(), arr.GlobalID, initiators, conn); err != nil { + return "", fmt.Errorf("[createHost] Failed to register host with connectivity %s on %s: %v", conn, arr.GlobalID, err) + } + if primaryArrayID == "" { + primaryArrayID = arr.GlobalID + } + } + + if primaryArrayID != "" { + log.Infof("[createHost] Success. Primary array: %s", primaryArrayID) + return primaryArrayID, nil + } + return "", fmt.Errorf("[createHost] Failed to register host on any array") +} + +func nodeMatchSelector(node *corev1.Node, selector *corev1.NodeSelector, conn gopowerstore.HostConnectivityEnum) (bool, error) { + runtimeSelector, err := nodeaffinity.NewNodeSelector(selector) + if err != nil { + return false, err + } + if runtimeSelector.Match(node) { + if conn != "" { + return false, fmt.Errorf("match expressions should be mutual exclusive, a duplicated match found") + } + return true, nil + } + return false, nil +} + +// register host +// For backwards compatibility to use array metro topology +func (s *Service) createHostMetroTopologyAndLocal( ctx context.Context, initiators []string, ) (string, error) { - nodeLabels, err := k8sutils.GetNodeLabels(context.Background(), s.opts.KubeConfigPath, s.opts.KubeNodeName) + log := log.WithContext(ctx) + nodeLabels, err := k8sutils.Kubeclient.GetNodeLabels(context.Background(), s.opts.KubeNodeName) if err != nil { return "", fmt.Errorf("failed to get node labels for node %s: %v", s.opts.KubeNodeName, err) } @@ -1880,13 +2236,9 @@ func (s *Service) createHost( return "", fmt.Errorf("[createHost] Failed to register host on any array") } -func (s *Service) handleLabelMatchRegistration( - ctx context.Context, - arr *array.PowerStoreArray, - initiators []string, - nodeLabels map[string]string, - arrayAddedList map[string]bool, +func (s *Service) handleLabelMatchRegistration(ctx context.Context, arr *array.PowerStoreArray, initiators []string, nodeLabels map[string]string, arrayAddedList map[string]bool, ) (bool, error) { + log := log.WithContext(ctx) // Early exit if no array labels match the node labels anyLabelMatch := false for _, configuredArr := range getArrayfn(s) { @@ -1990,6 +2342,7 @@ func (s *Service) handleNoLabelMatchRegistration( nodeLabels map[string]string, arrayAddedList map[string]bool, ) (bool, error) { + log := log.WithContext(ctx) // Early exit if no array labels match the node labels anyLabelMatch := false for _, configuredArr := range getArrayfn(s) { @@ -2086,6 +2439,7 @@ func (s *Service) isRemoteToOtherArray( ctx context.Context, arrA, arrB *array.PowerStoreArray, ) bool { + log := log.WithContext(ctx) // fetch arrA’s remotes remotesA, err := getAllRemoteSystemsFunc(arrA, ctx) if err != nil { @@ -2120,6 +2474,7 @@ func (s *Service) isRemoteToOtherArray( // Checks if host with given initiators already exists func (s *Service) isHostAlreadyRegistered(ctx context.Context, client gopowerstore.Client, initiators []string) bool { + log := log.WithContext(ctx) existingHosts, err := client.GetHosts(ctx) if err != nil { log.Warnf("[isHostAlreadyRegistered] Failed to get hosts: %v", err) @@ -2146,6 +2501,7 @@ func (s *Service) registerHost( initiators []string, connType gopowerstore.HostConnectivityEnum, ) error { + log := log.WithContext(ctx) description := fmt.Sprintf("k8s node: %s", s.opts.KubeNodeName) reqInitiators := s.buildInitiatorsArray(initiators, arrayID) osType := gopowerstore.OSTypeEnumLinux @@ -2200,6 +2556,7 @@ func labelsMatch(arrayLabels, nodeLabels map[string]string) bool { func (s *Service) modifyHostInitiators(ctx context.Context, hostID string, client gopowerstore.Client, initiatorsToAdd []string, initiatorsToDelete []string, initiatorsToModify []string, arrayID string, connectivity *gopowerstore.HostConnectivityEnum, ) error { + log := log.WithContext(ctx) if len(initiatorsToDelete) > 0 { modifyParams := gopowerstore.HostModify{RemoveInitiators: &initiatorsToDelete} _, err := client.ModifyHost(ctx, &modifyParams, hostID) @@ -2270,7 +2627,7 @@ func checkIQNS(IQNs []string, host gopowerstore.Host) (iqnToAdd, iqnToDelete []s iqnToDelete = append(iqnToDelete, iqn) } } - return + return iqnToAdd, iqnToDelete } func (s *Service) buildInitiatorsArrayModify(initiators []string, arrayID string) []gopowerstore.UpdateInitiatorInHost { @@ -2294,19 +2651,18 @@ func (s *Service) buildInitiatorsArrayModify(initiators []string, arrayID string func (s *Service) fileExists(filename string) bool { _, err := s.Fs.Stat(filename) + logFields := csmlog.Fields{ + "filename": filename, + "error": err, + } + log := log.WithFields(logFields) if err == nil { return true } if os.IsNotExist(err) { - log.WithFields(log.Fields{ - "filename": filename, - "error": err, - }).Error("File doesn't exist") + log.Error("File does not exist") } else { - log.WithFields(log.Fields{ - "filename": filename, - "error": err, - }).Error("Error while checking stat of file") + log.Error("Error while checking stat of the file") } return false } @@ -2331,3 +2687,88 @@ func ExtractPort(urlString string) (string, error) { return port, nil } + +// updateMetroToplogy updates the metro topology in the response +func updateMetroToplogy(arr *array.PowerStoreArray, nodeLabels map[string]string, resp *csi.NodeGetInfoResponse) { + if arr.HostConnectivity != nil { + labelSet := labels.Set(nodeLabels) + if ok, matchingLabels := metroMatchNodeSelectorTerms(arr.HostConnectivity.Local.NodeSelectorTerms, labelSet); ok { + maps.Copy(resp.AccessibleTopology.Segments, matchingLabels) + } + + if ok, matchingLabels := metroMatchNodeSelectorTerms(arr.HostConnectivity.Metro.ColocatedBoth.NodeSelectorTerms, labelSet); ok { + maps.Copy(resp.AccessibleTopology.Segments, matchingLabels) + } + + if ok, matchingLabels := metroMatchNodeSelectorTerms(arr.HostConnectivity.Metro.ColocatedLocal.NodeSelectorTerms, labelSet); ok { + maps.Copy(resp.AccessibleTopology.Segments, matchingLabels) + } + + if ok, matchingLabels := metroMatchNodeSelectorTerms(arr.HostConnectivity.Metro.ColocatedRemote.NodeSelectorTerms, labelSet); ok { + maps.Copy(resp.AccessibleTopology.Segments, matchingLabels) + } + } +} + +// metroMatchNodeSelectorTerms checks if the metro labels from the secret match the node selector terms +// Returns the matched labels if bool is true, else empty map +func metroMatchNodeSelectorTerms(terms []corev1.NodeSelectorTerm, nodeLabels map[string]string) (bool, map[string]string) { + for _, term := range terms { + matched := true + matchedLabels := make(map[string]string) + + for _, expr := range term.MatchExpressions { + nodeVal, exists := nodeLabels[expr.Key] + switch expr.Operator { + case corev1.NodeSelectorOpIn: + if !exists { + matched = false + break + } + if slices.Contains(expr.Values, nodeVal) { + matchedLabels[expr.Key] = nodeVal + } else { + matched = false + } + + case corev1.NodeSelectorOpNotIn: + if !exists { + continue + } + if slices.Contains(expr.Values, nodeVal) { + matched = false + } + + case corev1.NodeSelectorOpExists: + if exists { + matchedLabels[expr.Key] = nodeVal + } else { + matched = false + } + + case corev1.NodeSelectorOpDoesNotExist: + if exists { + matched = false + } + + default: + matched = false + } + + if !matched { + break + } + } + + // Kubernetes treats nodeSelectorTerms as OR — if one term matches, overall match is true. + if matched { + return true, matchedLabels + } + } + + return false, nil +} + +func isNodeConnectedToArray(ctx context.Context, kubeNodeID string, arr *array.PowerStoreArray) bool { + return arr.CheckConnectivity(ctx, kubeNodeID) +} diff --git a/pkg/node/node_connectivity_checker.go b/pkg/node/node_connectivity_checker.go index a7e0e275..a51a42c0 100644 --- a/pkg/node/node_connectivity_checker.go +++ b/pkg/node/node_connectivity_checker.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2022-2024 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2022-2026 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,7 +33,6 @@ import ( "github.com/dell/gonvme" "github.com/dell/gopowerstore" "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" ) // pollingFrequency in seconds @@ -44,6 +43,7 @@ var probeStatus *sync.Map // startAPIService reads nodes to array status periodically func (s *Service) startAPIService(ctx context.Context) { + log := log.WithContext(ctx) if !s.isPodmonEnabled { log.Info("podmon is not enabled") return @@ -54,7 +54,8 @@ func (s *Service) startAPIService(ctx context.Context) { } // apiRouter serves http requests -func (s *Service) apiRouter(_ context.Context) { +func (s *Service) apiRouter(ctx context.Context) { + log := log.WithContext(ctx) log.Infof("starting http server on port %s", identifiers.APIPort) // create a new mux router router := mux.NewRouter() @@ -156,6 +157,7 @@ func getArrayConnectivityStatus(w http.ResponseWriter, r *http.Request) { // startNodeToArrayConnectivityCheck starts connectivityTest as one goroutine for each array func (s *Service) startNodeToArrayConnectivityCheck(ctx context.Context) { + log := log.WithContext(ctx) log.Debug("startNodeToArrayConnectivityCheck called") probeStatus = new(sync.Map) // in case if we want to store the status of default array, uncomment below line @@ -172,6 +174,7 @@ func (s *Service) startNodeToArrayConnectivityCheck(ctx context.Context) { // testConnectivityAndUpdateStatus runs probe to test connectivity from node to array // updates probeStatus map[array]ArrayConnectivityStatus func (s *Service) testConnectivityAndUpdateStatus(ctx context.Context, array *array.PowerStoreArray, timeout time.Duration) { + log := log.WithContext(ctx) defer func() { if err := recover(); err != nil { log.Errorf("panic occurred in testConnectivityAndUpdateStatus: %s for array having %s", err, array.GlobalID) @@ -211,7 +214,8 @@ func (s *Service) testConnectivityAndUpdateStatus(ctx context.Context, array *ar } // nodeProbe function used to store the status of array -func (s *Service) nodeProbe(_ context.Context, array *array.PowerStoreArray) error { +func (s *Service) nodeProbe(ctx context.Context, array *array.PowerStoreArray) error { + log := log.WithContext(ctx) // try to get the host host, err := array.Client.GetHostByName(context.Background(), s.nodeID) // possibly NFS could be there. @@ -313,19 +317,29 @@ func (s *Service) populateTargetsInCache(array *array.PowerStoreArray) { return } + networkIDs := map[string]struct{}{} for _, address := range infoList { - nvmeIP := strings.Split(address.Portal, ":") - log.Info("Trying to discover NVMe target from portal ", nvmeIP[0]) - nvmeTargets, err := s.nvmeLib.DiscoverNVMeTCPTargets(nvmeIP[0], false) + // discovering with one portal returns all targets in the network + // so if we already discovered an address with this network ID, continue + if _, ok := networkIDs[address.NetworkID]; ok { + continue + } + + nvmeIP := strings.Split(address.Portal, ":")[0] + log.Infof("Trying to discover NVMe targets from portal %s on network %s", nvmeIP, address.NetworkID) + nvmeTargets, err := s.nvmeLib.DiscoverNVMeTCPTargets(nvmeIP, false) if err != nil { - log.Error("couldn't discover targets") + log.Errorf("discovering portal: %s: %v", nvmeIP, err) continue } for _, target := range nvmeTargets { otherTargets := s.nvmeTargets[array.GlobalID] s.nvmeTargets[array.GlobalID] = append(otherTargets, target.TargetNqn) } - break + + // mark this network ID as discovered so we don't discover another portal in the same network + // since it will return all the same target information already seen + networkIDs[address.NetworkID] = struct{}{} } } } else if !s.useFC[array.GlobalID] && !s.useNFS { @@ -340,13 +354,20 @@ func (s *Service) populateTargetsInCache(array *array.PowerStoreArray) { } var ipAddress string var iscsiTargets []goiscsi.ISCSITarget + networkIDs := map[string]struct{}{} for _, address := range infoList { + // discovering with one portal returns all targets in the network + // so if we already discovered an address with this network ID, continue + if _, ok := networkIDs[address.NetworkID]; ok { + continue + } + // first check if this portal is reachable from this machine or not if ReachableEndPoint(address.Portal) { ipAddressList := splitIPAddress(address.Portal) ipAddress = ipAddressList[0] // doesn't matter how many portals are present, discovering from any one will list out all targets - log.Info("Trying to discover iSCSI target from portal ", ipAddress) + log.Infof("Trying to discover iSCSI target from portal %s ", ipAddress) ipInterface, err := s.iscsiLib.GetInterfaceForTargetIP(ipAddress) if err != nil { log.Errorf("couldn't get interface: %s", err.Error()) @@ -361,7 +382,10 @@ func (s *Service) populateTargetsInCache(array *array.PowerStoreArray) { otherTargets := s.iscsiTargets[array.GlobalID] s.iscsiTargets[array.GlobalID] = append(otherTargets, target.Target) } - break + + // mark this network ID as discovered so we don't discover another portal in the same network + // since it will return all the same target information already seen + networkIDs[address.NetworkID] = struct{}{} } log.Debugf("Portal is not rechable from the node") } diff --git a/pkg/node/node_connectivity_checker_test.go b/pkg/node/node_connectivity_checker_test.go index 08a08e49..f7e480dc 100644 --- a/pkg/node/node_connectivity_checker_test.go +++ b/pkg/node/node_connectivity_checker_test.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2022-2024 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2022-2026 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -162,6 +162,78 @@ func TestPopulateTargetsInCache(t *testing.T) { } }) + t.Run("PopulateTargetsInCache - nvmeTargets should be populated in multiple networks [NVMeTCP]", func(t *testing.T) { + setVariables(withMockNumberOfNVMeTCPTargets(2)) + nodeSvc.useNVME[firstGlobalID] = true + + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW1", + }, + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW1", + }, + { + Address: "192.168.2.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW2", + }, + { + Address: "192.168.2.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW2", + }, + }, nil) + + nodeSvc.populateTargetsInCache(nodeSvc.Arrays()[firstValidIP]) + if len(nodeSvc.nvmeTargets[firstGlobalID]) != 4 { + t.Errorf("Expected nvmeTargets to be populated") + } + }) + + t.Run("PopulateTargetsInCache - iscsiTargets should be populated in multiple networks [iSCSI]", func(t *testing.T) { + setVariables(withMockNumberOfISCSITargets(4)) + nodeSvc.useNVME[firstGlobalID] = false + + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW1", + }, + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW1", + }, + { + Address: "192.168.2.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW2", + }, + { + Address: "192.168.2.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW2", + }, + }, nil) + + nodeSvc.populateTargetsInCache(nodeSvc.Arrays()[firstValidIP]) + if len(nodeSvc.iscsiTargets[firstGlobalID]) != 4 { + t.Errorf("Expected iscsiTargets to be populated") + } + }) + t.Run("PopulateTargetsInCache - nvmeTargets should be populated [NVMeFC]", func(t *testing.T) { setVariables() nodeSvc.useNVME[firstGlobalID] = true diff --git a/pkg/node/node_test.go b/pkg/node/node_test.go index 413fe80b..9f516ff8 100644 --- a/pkg/node/node_test.go +++ b/pkg/node/node_test.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,16 +27,15 @@ import ( "os" "path" "path/filepath" + "strconv" "testing" - // "github.com/onsi/ginkgo/reporters" - - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/mocks" "github.com/dell/csi-powerstore/v2/pkg/array" "github.com/dell/csi-powerstore/v2/pkg/controller" "github.com/dell/csi-powerstore/v2/pkg/identifiers" "github.com/dell/csi-powerstore/v2/pkg/identifiers/k8sutils" + "github.com/dell/csmlog" "github.com/dell/gobrick" csictx "github.com/dell/gocsi/context" "github.com/dell/gofsutil" @@ -45,54 +44,60 @@ import ( "github.com/dell/gopowerstore" "github.com/dell/gopowerstore/api" gopowerstoremock "github.com/dell/gopowerstore/mocks" + "github.com/container-storage-interface/spec/lib/go/csi" ginkgo "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" gomega "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + corev1 "k8s.io/api/core/v1" + k8score "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" ) var ( - iscsiConnectorMock *mocks.ISCSIConnector - nvmeConnectorMock *mocks.NVMEConnector - fcConnectorMock *mocks.FcConnector - utilMock *mocks.UtilInterface - fsMock *mocks.FsInterface - nodeSvc *Service - clientMock *gopowerstoremock.Client - ctrlMock *mocks.ControllerInterface - iscsiLibMock *goiscsi.MockISCSI - nvmeLibMock *gonvme.MockNVMe - nodeLabelsRetrieverMock *mocks.NodeLabelsRetrieverInterface - nodeLabelsModifierMock *mocks.NodeLabelsModifierInterface + iscsiConnectorMock *mocks.ISCSIConnector + nvmeConnectorMock *mocks.NVMEConnector + fcConnectorMock *mocks.FcConnector + utilMock *mocks.UtilInterface + fsMock *mocks.FsInterface + nodeSvc *Service + clientMock *gopowerstoremock.Client + ctrlMock *mocks.ControllerInterface + iscsiLibMock *goiscsi.MockISCSI + nvmeLibMock *gonvme.MockNVMe ) const ( - validBaseVolumeID = "39bb1b5f-5624-490d-9ece-18f7b28a904e" - validBlockVolumeID = "39bb1b5f-5624-490d-9ece-18f7b28a904e/gid1/scsi" - validClusterName = "localSystemName" - validNfsVolumeID = "39bb1b5f-5624-490d-9ece-18f7b28a904e/gid2/nfs" - validRemoteVolID = "9f840c56-96e6-4de9-b5a3-27e7c20eaa77" - invalidBlockVolumeID = "39bb1b5f-5624-490d-9ece-18f7b28a904e/gid3/scsi" - validVolSize = 16 * 1024 * 1024 * 1024 - validLUNID = "3" - validLUNIDINT = 3 - nodeStagePrivateDir = "test/stage" - unimplementedErrMsg = "rpc error: code = Unimplemented desc = " - validNodeID = "csi-node-1a47a1b91c444a8a90193d8066669603-127.0.0.1" - validHostID = "e8f4c5f8-c2fc-4df4-bd99-c292c12b55be" - validHostName = "csi-node-1a47a1b91c444a8a90193d8066669603" - testErrMsg = "test err" - validDeviceWWN = "68ccf09800e23ab798312a05426acae0" - validDevPath = "/dev/sdag" - validDevName = "sdag" - validNfsExportPath = "/mnt/nfs" - validTargetPath = "/var/lib/kubelet/pods/dac33335-a31d-11e9-b46e-005056917428/" + + validBaseVolumeID = "39bb1b5f-5624-490d-9ece-18f7b28a904e" + validRemoteBaseVolumeID = "00000000-0000-0000-0000-000000000002" + validClusterName = "localSystemName" + validNfsVolumeID = "39bb1b5f-5624-490d-9ece-18f7b28a904e/gid2/nfs" + validRemoteVolID = "9f840c56-96e6-4de9-b5a3-27e7c20eaa77" + invalidBlockVolumeID = "39bb1b5f-5624-490d-9ece-18f7b28a904e/gid3/scsi" + validVolSize = 16 * 1024 * 1024 * 1024 + validLUNID = "3" + validLUNIDINT = 3 + nodeStagePrivateDir = "test/stage" + validNodeID = "csi-node-1a47a1b91c444a8a90193d8066669603-127.0.0.1" + validNodeID2 = "csi-node-90193d80666696031a47a1b91c444a8a-127.0.0.1" + validHostID = "e8f4c5f8-c2fc-4df4-bd99-c292c12b55be" + validHostName = "csi-node-1a47a1b91c444a8a90193d8066669603" + validDeviceWWN = "68ccf09800e23ab798312a05426acae0" + validDevName = "sdag" + validNfsExportPath = "/mnt/nfs" + validTargetPath = "/var/lib/kubelet/pods/dac33335-a31d-11e9-b46e-005056917428/" + "volumes/kubernetes.io~csi/csi-d91431aba3/mount" validStagingPath = "/var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/" + "staging/csi-44b46e98ae/c875b4f0-172e-4238-aec7-95b379eb55db" firstValidIP = "gid1" secondValidIP = "gid2" + metroFirstValidIP = "gid3" + metroSecondValidIP = "gid4" firstGlobalID = "unique1" secondGlobalID = "unique2" validNasName = "my-nas-name" @@ -101,6 +106,22 @@ const ( validEphemeralName = "ephemeral-39bb1b5f-5624-490d-9ece-18f7b28a904e/gid1/scsi" ephemerallockfile = "/var/lib/kubelet/plugins/kubernetes.io/csi/pv/ephemeral/39bb1b5f-5624-490d-9ece-18f7b28a904e/gid1/scsi/id" validMetroSessionID = "9abd0198-2733-4e46-b5fa-456e9c367184" + ProtoSCSI = "scsi" + + zoneLabelKey = "topology.kubernetes.io/zone" + zone1LabelValue = "zone1" + zone2LabelValue = "zone2" +) + +var ( + // format: // + validBlockVolumeHandle = filepath.Join(validBaseVolumeID, firstValidIP, ProtoSCSI) + + // format: //:/ + validMetroVolumeHandle = filepath.Join(validBlockVolumeHandle+":"+validRemoteBaseVolumeID, secondValidIP) + + zone1Label = map[string]string{zoneLabelKey: zone1LabelValue} + zone2Label = map[string]string{zoneLabelKey: zone2LabelValue} ) var ( @@ -214,6 +235,21 @@ func setFSmocks() { } func TestCSINodeService(t *testing.T) { + defaultK8sConfigFunc := k8sutils.InClusterConfigFunc + defaultK8sClientsetFunc := k8sutils.NewForConfigFunc + + k8sutils.InClusterConfigFunc = func() (*rest.Config, error) { + return &rest.Config{}, nil + } + k8sutils.NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return fake.NewClientset(), nil + } + + defer func() { + k8sutils.InClusterConfigFunc = defaultK8sConfigFunc + k8sutils.NewForConfigFunc = defaultK8sClientsetFunc + }() + gomega.RegisterFailHandler(ginkgo.Fail) junitReporter := reporters.NewJUnitReporter("node-svc.xml") ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "CSINodeService testing suite", []ginkgo.Reporter{junitReporter}) @@ -250,7 +286,135 @@ func getTestArrays() map[string]*array.PowerStoreArray { return arrays } -func setVariables() { +func getMetroTestArrays() map[string]*array.PowerStoreArray { + arrays := make(map[string]*array.PowerStoreArray) + first := &array.PowerStoreArray{ + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array3", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone2"}, + }, + }, + }, + }, + }, + }, + }, + IP: metroFirstValidIP, + Client: clientMock, + } + second := &array.PowerStoreArray{ + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array4", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone2"}, + }, + }, + }, + }, + }, + }, + }, + IP: metroSecondValidIP, + Client: clientMock, + } + + arrays[metroFirstValidIP] = first + arrays[metroSecondValidIP] = second + + return arrays +} + +func getBaseClient() *k8sutils.K8sClient { + return &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone": "zone1"}, + }, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node2", + Labels: map[string]string{"topology.kubernetes.io/zone": "zone2"}, + }, + }, + }...), + } +} + +type variableOptions struct { + mockNumberOfNVMeTCPTargets int + mockNumberOfISCSITargets int +} + +type variableOption func(*variableOptions) + +func withMockNumberOfNVMeTCPTargets(count int) variableOption { + return func(vo *variableOptions) { + vo.mockNumberOfNVMeTCPTargets = count + } +} + +func withMockNumberOfISCSITargets(count int) variableOption { + return func(vo *variableOptions) { + vo.mockNumberOfISCSITargets = count + } +} + +func setVariables(options ...variableOption) { + option := &variableOptions{} + for _, vo := range options { + vo(option) + } + + mockNVMeOptions := make(map[string]string) + mockISCSIOptions := make(map[string]string) + if option.mockNumberOfNVMeTCPTargets != 0 { + mockNVMeOptions[gonvme.MockNumberOfTCPTargets] = strconv.Itoa(option.mockNumberOfNVMeTCPTargets) + } + if option.mockNumberOfISCSITargets != 0 { + mockISCSIOptions[goiscsi.MockNumberOfTargets] = strconv.Itoa(option.mockNumberOfISCSITargets) + } + iscsiConnectorMock = new(mocks.ISCSIConnector) nvmeConnectorMock = new(mocks.NVMEConnector) fcConnectorMock = new(mocks.FcConnector) @@ -258,12 +422,8 @@ func setVariables() { fsMock = new(mocks.FsInterface) ctrlMock = new(mocks.ControllerInterface) clientMock = new(gopowerstoremock.Client) - iscsiLibMock = goiscsi.NewMockISCSI(nil) - nvmeLibMock = gonvme.NewMockNVMe(nil) - nodeLabelsRetrieverMock = new(mocks.NodeLabelsRetrieverInterface) - k8sutils.NodeLabelsRetriever = nodeLabelsRetrieverMock - nodeLabelsModifierMock = new(mocks.NodeLabelsModifierInterface) - k8sutils.NodeLabelsModifier = nodeLabelsModifierMock + iscsiLibMock = goiscsi.NewMockISCSI(mockISCSIOptions) + nvmeLibMock = gonvme.NewMockNVMe(mockNVMeOptions) arrays := getTestArrays() nodeSvc = &Service{ @@ -277,7 +437,13 @@ func setVariables() { nodeID: validNodeID, initialized: true, isPodmonEnabled: false, + opts: Opts{ + KubeNodeName: "node1", + }, } + + k8sutils.Kubeclient = getBaseClient() + nodeSvc.iscsiTargets = make(map[string][]string) nodeSvc.nvmeTargets = make(map[string][]string) nodeSvc.useFC = make(map[string]bool) @@ -295,18 +461,17 @@ func setVariables() { } func setDefaultNodeLabelsMock() { - nodeLabelsRetrieverMock.On("BuildConfigFromFlags", mock.Anything, mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("GetNodeLabels", mock.Anything, mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("InClusterConfig", mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("NewForConfig", mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("GetNVMeUUIDs", mock.Anything).Return(nil, nil) - nodeLabelsModifierMock.On("AddNVMeLabels", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) } +var options []variableOption + var _ = ginkgo.Describe("CSINodeService", func() { + os.Setenv(identifiers.EnvKubeNodeName, "node1") + ginkgo.BeforeEach(func() { - setVariables() + setVariables(options...) }) + nasData := []gopowerstore.NAS{ { NfsServers: []gopowerstore.NFSServerInstance{ @@ -349,6 +514,8 @@ var _ = ginkgo.Describe("CSINodeService", func() { }}, Name: "host-name", }}, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, nil) clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) @@ -365,6 +532,8 @@ var _ = ginkgo.Describe("CSINodeService", func() { ginkgo.It("should fail", func() { nodeSvc.nodeID = "" + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, nil) fsMock.On("ReadFile", mock.Anything).Return([]byte("my-host-id"), errors.New("no such file")) setDefaultNodeLabelsMock() @@ -404,6 +573,8 @@ var _ = ginkgo.Describe("CSINodeService", func() { }}, Name: "host-name", }}, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, nil) clientMock.On("CreateHost", mock.Anything, mock.Anything). Return(gopowerstore.CreateResponse{ID: validHostID}, nil) setDefaultNodeLabelsMock() @@ -444,6 +615,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { }}, Name: "host-name", }}, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) clientMock.On("CreateHost", mock.Anything, mock.Anything). Return(gopowerstore.CreateResponse{ID: validHostID}, nil) setDefaultNodeLabelsMock() @@ -488,6 +660,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { clientMock.On("ModifyHost", mock.Anything, mock.Anything, "host-id"). Return(gopowerstore.CreateResponse{ID: "host-id"}, nil) setDefaultNodeLabelsMock() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) err := nodeSvc.Init() gomega.Expect(err).To(gomega.BeNil()) @@ -528,6 +701,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { clientMock.On("ModifyHost", mock.Anything, mock.Anything, "host-id"). Return(gopowerstore.CreateResponse{ID: "host-id"}, nil) setDefaultNodeLabelsMock() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) err := nodeSvc.Init() gomega.Expect(err).To(gomega.BeNil()) @@ -579,6 +753,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { clientMock.On("CreateHost", mock.Anything, mock.Anything). Return(gopowerstore.CreateResponse{ID: validHostID}, nil) setDefaultNodeLabelsMock() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) err := nodeSvc.Init() gomega.Expect(err).To(gomega.BeNil()) @@ -625,6 +800,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { clientMock.On("CreateHost", mock.Anything, mock.Anything). Return(gopowerstore.CreateResponse{ID: validHostID}, nil) setDefaultNodeLabelsMock() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) err := nodeSvc.Init() gomega.Expect(err).To(gomega.BeNil()) @@ -649,13 +825,6 @@ var _ = ginkgo.Describe("CSINodeService", func() { nodeSvc.opts.KubeNodeName = identifiers.EnvKubeNodeName nodeSvc.opts.KubeConfigPath = identifiers.EnvKubeConfigPath - nodeLabelsRetrieverMock.On("GetNVMeUUIDs", mock.Anything).Return( - map[string]string{ - "node1": "duplicate-uuid", - "node2": "duplicate-uuid", - }, - nil, - ) clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")). Return(gopowerstore.Host{}, gopowerstore.APIError{ @@ -679,6 +848,10 @@ var _ = ginkgo.Describe("CSINodeService", func() { clientMock.On("CreateHost", mock.Anything, mock.Anything). Return(gopowerstore.CreateResponse{ID: validHostID}, nil) setDefaultNodeLabelsMock() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) + + k8sutils.Kubeclient.SetNodeLabel(context.Background(), "node1", "hostnqn-uuid", "duplicate-uuid") + k8sutils.Kubeclient.SetNodeLabel(context.Background(), "node2", "hostnqn-uuid", "duplicate-uuid") err := nodeSvc.Init() gomega.Expect(err).To(gomega.BeNil()) @@ -721,6 +894,8 @@ var _ = ginkgo.Describe("CSINodeService", func() { clientMock.On("CreateHost", mock.Anything, mock.Anything). Return(gopowerstore.CreateResponse{ID: validHostID}, nil) setDefaultNodeLabelsMock() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, nil) err := nodeSvc.Init() gomega.Expect(err).To(gomega.BeNil()) @@ -765,6 +940,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { clientMock.On("CreateHost", mock.Anything, mock.Anything). Return(gopowerstore.CreateResponse{ID: validHostID}, nil) setDefaultNodeLabelsMock() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) err := nodeSvc.Init() gomega.Expect(err).To(gomega.BeNil()) @@ -813,6 +989,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { setDefaultNodeLabelsMock() nodeSvc.Arrays()[firstValidIP].BlockProtocol = "default_protocol" + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) err := nodeSvc.Init() gomega.Expect(err).To(gomega.BeNil()) @@ -834,6 +1011,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { Return([]string{}, nil) fcConnectorMock.On("GetInitiatorPorts", mock.Anything). Return([]string{}, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) err := nodeSvc.Init() gomega.Expect(err).To(gomega.BeNil()) @@ -875,6 +1053,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { Message: "not found", }, }) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) nodeSvc.useNFS = true arrays := getTestArrays() err := nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) @@ -1135,11 +1314,15 @@ var _ = ginkgo.Describe("CSINodeService", func() { }, Name: "host-name", }, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) arrays := getTestArrays() + err := nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("no active iscsi sessions")) + nodeSvc.iscsiTargets[firstGlobalID] = []string{"iqn.2015-10.com.dell:dellemc-foobar-123-a-7ceb34a0"} nodeSvc.startNodeToArrayConnectivityCheck(context.Background()) - err := nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) + err = nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) gomega.Expect(err).To(gomega.BeNil()) }) }) @@ -1173,7 +1356,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { }, Name: "host-name", }, nil) - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) arrays := getTestArrays() if nodeSvc.useNVME[firstGlobalID] { nodeSvc.useNVME[firstGlobalID] = false @@ -1197,7 +1380,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { Initiators: []gopowerstore.InitiatorInstance{}, Name: "host-name", }, nil) - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) arrays := getTestArrays() if nodeSvc.useNVME[firstGlobalID] { nodeSvc.useNVME[firstGlobalID] = false @@ -1221,7 +1404,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) scsiStageVolumeOK(utilMock, fsMock) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( @@ -1241,7 +1424,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { scsiStageVolumeOK(utilMock, fsMock) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( @@ -1260,7 +1443,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { scsiStageVolumeOK(utilMock, fsMock) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( @@ -1282,7 +1465,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("Chmod", filepath.Join(stagingPath, commonNfsVolumeFolder), os.ModeSticky|os.ModePerm).Return(nil) fsMock.On("GetUtil").Return(utilMock) - publishContext := getValidPublishContext() + publishContext := make(map[string]string) publishContext["NfsExportPath"] = validNfsExportPath res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ @@ -1311,7 +1494,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("Chmod", filepath.Join(stagingPath, commonNfsVolumeFolder), os.ModeSticky|os.ModePerm).Return(nil) fsMock.On("GetUtil").Return(utilMock) - publishContext := getValidPublishContext() + publishContext := make(map[string]string) publishContext["NfsExportPath"] = validNfsExportPath publishContext[identifiers.KeyNasName] = validNasName publishContext[identifiers.KeyNfsACL] = "0777" @@ -1325,6 +1508,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { clientMock.On("GetNfsServer", mock.Anything, validNasName).Return(gopowerstore.NFSServerInstance{ID: validNfsServerID, IsNFSv4Enabled: true}, nil) clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID, NfsServers: nfsServers}, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) nfsv4ACLsMock.On("SetNfsv4Acls", mock.Anything, mock.Anything).Return(nil) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ @@ -1353,7 +1537,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("Chmod", filepath.Join(stagingPath, commonNfsVolumeFolder), os.ModeSticky|os.ModePerm).Return(nil) fsMock.On("GetUtil").Return(utilMock) - publishContext := getValidPublishContext() + publishContext := make(map[string]string) publishContext["NfsExportPath"] = validNfsExportPath publishContext[identifiers.KeyNfsACL] = "A::OWNER@:RWX" @@ -1367,7 +1551,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { nfsv4ACLsMock.On("SetNfsv4Acls", mock.Anything, mock.Anything).Return(nil) clientMock.On("GetNASByName", mock.Anything, "").Return(gopowerstore.NAS{ID: validNasID, NfsServers: nfsServers}, nil) clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{ID: validNfsServerID, IsNFSv4Enabled: true}, nil) - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: validNfsVolumeID, PublishContext: publishContext, @@ -1379,21 +1563,137 @@ var _ = ginkgo.Describe("CSINodeService", func() { }) ginkgo.When("using iSCSI for Metro volume", func() { - ginkgo.It("should successfully stage Metro iSCSI volume", func() { - setDefaultClientMocks() - iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil).Times(4) - scsiStageVolumeOK(utilMock, fsMock) - scsiStageRemoteMetroVolumeOK(utilMock, fsMock) - metroVolumeID := fmt.Sprintf("%s:%s/%s", validBlockVolumeID, validRemoteVolID, secondValidIP) - res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: metroVolumeID, - PublishContext: getValidRemoteMetroPublishContext(), - StagingTargetPath: nodeStagePrivateDir, - VolumeCapability: getCapabilityWithVoltypeAccessFstype( - "mount", "single-writer", "ext4"), + ginkgo.When("hostConnectivity is not configured in the secret (backward compatibility)", func() { + ginkgo.It("should successfully stage Metro iSCSI volume", func() { + setDefaultClientMocks() + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil).Times(4) + scsiStageVolumeOK(utilMock, fsMock) + scsiStageRemoteMetroVolumeOK(utilMock, fsMock) + metroVolumeID := fmt.Sprintf("%s:%s/%s", validBlockVolumeHandle, validRemoteVolID, secondValidIP) + res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: metroVolumeID, + PublishContext: getValidUniformMetroPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "mount", "single-writer", "ext4"), + }) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) + }) + }) + + ginkgo.When("hostConnectivity is configured for non-uniform metro", func() { + defaultNodeID := nodeSvc.nodeID + ginkgo.BeforeEach(func() { + arrays := getTestArrays() + arrays[firstValidIP].HostConnectivity = &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: zoneLabelKey, + Operator: "In", + Values: []string{zone1LabelValue}, + }, + }, + }, + }, + }, + } + arrays[secondValidIP].HostConnectivity = &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: zoneLabelKey, + Operator: "In", + Values: []string{zone2LabelValue}, + }, + }, + }, + }, + }, + } + nodeSvc.SetArrays(arrays) + nodeSvc.SetDefaultArray(arrays[firstValidIP]) + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: zone1Label, + Annotations: map[string]string{identifiers.KeyNodeID: "{" + strconv.Quote(identifiers.Name) + ":" + strconv.Quote(validNodeID) + "}"}, + }, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID2, + Labels: zone2Label, + Annotations: map[string]string{identifiers.KeyNodeID: "{" + strconv.Quote(identifiers.Name) + ":" + strconv.Quote(validNodeID2) + "}"}, + }, + }, + }...), + } + }) + ginkgo.AfterEach(func() { + // resetting for other tests, since we're all using the same instance + arrays := getTestArrays() + nodeSvc.SetArrays(arrays) + nodeSvc.SetDefaultArray(arrays[firstValidIP]) + nodeSvc.nodeID = defaultNodeID + k8sutils.Kubeclient = getBaseClient() + }) + + ginkgo.It("should not stage local volume when local array no connectivity to node", func() { + // publish to the "remote" node + nodeSvc.nodeID = validNodeID2 + + clientMock.On("GetVolume", mock.Anything, validBaseVolumeID).Return(gopowerstore.Volume{ + ID: validBaseVolID, + MetroReplicationSessionID: validMetroSessionID, + }, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, validMetroSessionID).Return(gopowerstore.ReplicationSession{ + ID: validMetroSessionID, + State: gopowerstore.RsStateOk, + LocalResourceState: string(gopowerstore.ReplicationResourceStatePromoted), + }, nil) + + setDefaultClientMocks() + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil).Times(2) + scsiStageRemoteMetroVolumeOK(utilMock, fsMock) + metroVolumeID := fmt.Sprintf("%s:%s/%s", validBlockVolumeHandle, validRemoteVolID, secondValidIP) + req := &csi.NodeStageVolumeRequest{ + VolumeId: metroVolumeID, + // do not include identifiers.TargetMapDeviceWWN in publish context + PublishContext: getValidRemoteMetroPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "mount", "single-writer", "ext4"), + } + resp, err := nodeSvc.NodeStageVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(resp).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) + }) + + ginkgo.It("should not stage remote metro volume remote array has no connectivity to node", func() { + setDefaultClientMocks() + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil).Times(2) + scsiStageVolumeOK(utilMock, fsMock) + metroVolumeID := fmt.Sprintf("%s:%s/%s", validBlockVolumeHandle, validRemoteVolID, secondValidIP) + req := &csi.NodeStageVolumeRequest{ + VolumeId: metroVolumeID, + // do not include identifiers.TargetMapRemoteDeviceWWN in publish context + PublishContext: getValidPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "mount", "single-writer", "ext4"), + } + resp, err := nodeSvc.NodeStageVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(resp).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) }) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(res).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) }) }) @@ -1413,7 +1713,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("GetDiskFormat", mock.Anything, stagingPath).Return("", nil) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( @@ -1437,7 +1737,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) fsMock.On("GetUtil").Return(utilMock) utilMock.On("GetDiskFormat", mock.Anything, stagingPath).Return("", nil) - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: validNfsVolumeID, PublishContext: publishContext, @@ -1453,7 +1753,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { ginkgo.When("missing volume capabilities", func() { ginkgo.It("should fail", func() { req := &csi.NodeStageVolumeRequest{} - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) res, err := nodeSvc.NodeStageVolume(context.Background(), req) gomega.Expect(res).To(gomega.BeNil()) gomega.Expect(err).NotTo(gomega.BeNil()) @@ -1467,7 +1767,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "single-writer", "ext4"), StagingTargetPath: nodeStagePrivateDir, } - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) res, err := nodeSvc.NodeStageVolume(context.Background(), req) gomega.Expect(res).To(gomega.BeNil()) gomega.Expect(err).NotTo(gomega.BeNil()) @@ -1482,7 +1782,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { StagingTargetPath: nodeStagePrivateDir, VolumeId: invalidBlockVolumeID, } - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) res, err := nodeSvc.NodeStageVolume(context.Background(), req) gomega.Expect(res).To(gomega.BeNil()) gomega.Expect(err).NotTo(gomega.BeNil()) @@ -1494,9 +1794,9 @@ var _ = ginkgo.Describe("CSINodeService", func() { ginkgo.It("should fail", func() { req := &csi.NodeStageVolumeRequest{ VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "single-writer", "ext4"), - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, } - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) res, err := nodeSvc.NodeStageVolume(context.Background(), req) gomega.Expect(res).To(gomega.BeNil()) gomega.Expect(err).NotTo(gomega.BeNil()) @@ -1536,7 +1836,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("Remove", stagingPath).Return(nil).Once() res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( @@ -1547,6 +1847,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { }) ginkgo.When("unstaging fails", func() { + setVariables() setDefaultClientMocks() ginkgo.It("should fail", func() { setDefaultClientMocks() @@ -1556,7 +1857,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("IsDeviceOrResourceBusy", e).Return(false) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( @@ -1573,9 +1874,14 @@ var _ = ginkgo.Describe("CSINodeService", func() { ginkgo.It("should fail [deviceWWN]", func() { setDefaultClientMocks() setFSmocks() + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true + } + defer func() { isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc }() iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: map[string]string{ identifiers.TargetMapLUNAddress: validLUNID, }, @@ -1588,30 +1894,12 @@ var _ = ginkgo.Describe("CSINodeService", func() { gomega.Expect(err.Error()).To(gomega.ContainSubstring("deviceWWN must be in publish context")) }) - ginkgo.It("should fail [volumeLUNAddress]", func() { - setDefaultClientMocks() - setFSmocks() - iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) - res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, - PublishContext: map[string]string{ - identifiers.TargetMapDeviceWWN: validDeviceWWN, - }, - StagingTargetPath: nodeStagePrivateDir, - VolumeCapability: getCapabilityWithVoltypeAccessFstype( - "mount", "single-writer", "ext4"), - }) - gomega.Expect(err).ToNot(gomega.BeNil()) - gomega.Expect(res).To(gomega.BeNil()) - gomega.Expect(err.Error()).To(gomega.ContainSubstring("volumeLUNAddress must be in publish context")) - }) - ginkgo.It("should fail [iscsiTargets]", func() { setDefaultClientMocks() setFSmocks() iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) _, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: map[string]string{ identifiers.TargetMapDeviceWWN: validDeviceWWN, identifiers.TargetMapLUNAddress: validLUNID, @@ -1630,7 +1918,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { nodeSvc.useNVME[firstGlobalID] = true nodeSvc.useFC[firstGlobalID] = true _, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: map[string]string{ identifiers.TargetMapDeviceWWN: validDeviceWWN, identifiers.TargetMapLUNAddress: validLUNID, @@ -1649,7 +1937,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { nodeSvc.useNVME[firstGlobalID] = true nodeSvc.useFC[firstGlobalID] = false _, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: map[string]string{ identifiers.TargetMapDeviceWWN: validDeviceWWN, identifiers.TargetMapLUNAddress: validLUNID, @@ -1667,7 +1955,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fcConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) nodeSvc.useFC[firstGlobalID] = true _, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: map[string]string{ identifiers.TargetMapDeviceWWN: validDeviceWWN, identifiers.TargetMapLUNAddress: validLUNID, @@ -1688,7 +1976,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { scsiStageVolumeOK(utilMock, fsMock) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( @@ -1713,7 +2001,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("BindMount", mock.Anything, "/dev", filepath.Join(nodeStagePrivateDir, validBaseVolumeID)).Return(e) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( @@ -1746,7 +2034,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("MkdirAll", stagingPath, mock.Anything).Return(errors.New("some-error")) res, err := nodeSvc.NodeStageVolume(context.Background(), req) - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) gomega.Expect(err).ToNot(gomega.BeNil()) gomega.Expect(res).To(gomega.BeNil()) gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't create target folder")) @@ -1761,7 +2049,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("Mount", mock.Anything, validNfsExportPath, stagingPath, "").Return(errors.New("some-error")) res, err := nodeSvc.NodeStageVolume(context.Background(), req) - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) gomega.Expect(err).ToNot(gomega.BeNil()) gomega.Expect(res).To(gomega.BeNil()) gomega.Expect(err.Error()).To(gomega.ContainSubstring("error mount nfs share")) @@ -1777,7 +2065,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("MkdirAll", filepath.Join(stagingPath, commonNfsVolumeFolder), mock.Anything).Return(errors.New("some-error")) res, err := nodeSvc.NodeStageVolume(context.Background(), req) - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) gomega.Expect(err).ToNot(gomega.BeNil()) gomega.Expect(res).To(gomega.BeNil()) gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't create common folder")) @@ -1794,7 +2082,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("Chmod", filepath.Join(stagingPath, commonNfsVolumeFolder), os.ModeSticky|os.ModePerm).Return(errors.New("some-error")) res, err := nodeSvc.NodeStageVolume(context.Background(), req) - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) gomega.Expect(err).ToNot(gomega.BeNil()) gomega.Expect(res).To(gomega.BeNil()) gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't change permissions of folder")) @@ -1812,7 +2100,12 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("MkdirAll", filepath.Join(stagingPath, commonNfsVolumeFolder), mock.Anything).Return(nil) fsMock.On("Chmod", filepath.Join(stagingPath, commonNfsVolumeFolder), os.ModeSticky|os.ModePerm).Return(nil) clientMock.On("ModifyNFSExport", mock.Anything, mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{}, errors.New("some-error")) - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true + } + defer func() { isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc }() publishContext := getValidPublishContext() publishContext["NfsExportPath"] = validNfsExportPath publishContext["allowRoot"] = "false" @@ -1831,57 +2124,273 @@ var _ = ginkgo.Describe("CSINodeService", func() { gomega.Expect(err.Error()).To(gomega.ContainSubstring("failure when modifying nfs export")) }) }) - }) - - ginkgo.Describe("calling NodeUnstage()", func() { - stagingPath := filepath.Join(nodeStagePrivateDir, validBaseVolumeID) - - ginkgo.When("unstaging block volume", func() { - ginkgo.It("should succeed [iSCSI]", func() { - mountInfo := []gofsutil.Info{ - { - Device: validDevName, - Path: stagingPath, - }, - } - fsMock.On("GetUtil").Return(utilMock) - fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) - fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) - - utilMock.On("Unmount", mock.Anything, stagingPath).Return(nil) + ginkgo.When("when a uniform metro session is fractured", func() { + var defaultCreateOrUpdateJournalEntryFunc func(ctx context.Context, name string, volumeHandle array.VolumeHandle, deferredArrayID string, nodeName string, operation string, request []byte) error + var defaultCheckMetroStateFunc func(ctx context.Context, volumeHandle array.VolumeHandle, client gopowerstore.Client, client2 gopowerstore.Client) (*array.MetroFracturedResponse, bool, error) - fsMock.On("Remove", stagingPath).Return(nil) - fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0o640)).Return(nil) + ginkgo.BeforeEach(func() { + defaultCreateOrUpdateJournalEntryFunc = createOrUpdateJournalEntryFunc + defaultCheckMetroStateFunc = checkMetroStateFunc + }) + ginkgo.AfterEach(func() { + // resetting for other tests, since we're all using the same instance + createOrUpdateJournalEntryFunc = defaultCreateOrUpdateJournalEntryFunc + checkMetroStateFunc = defaultCheckMetroStateFunc + }) - iscsiConnectorMock.On("DisconnectVolumeByDeviceName", mock.Anything, validDevName).Return(nil) + ginkgo.It("should create a journal entry when remote is not staged", func() { + setDefaultClientMocks() + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true + } + defer func() { isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc }() + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) + scsiStageVolumeOK(utilMock, fsMock) - fsMock.On("Remove", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID)).Return(nil) - fsMock.On("IsNotExist", mock.Anything).Return(false) + // RemoteVolume get fails. So remoteVolume stage fails. + clientMock.On("GetVolume", mock.Anything, "00000000-0000-0000-0000-000000000002").Return(gopowerstore.Volume{}, errors.New("some-error")) + checkMetroStateFunc = func(_ context.Context, _ array.VolumeHandle, _ gopowerstore.Client, _ gopowerstore.Client) (*array.MetroFracturedResponse, bool, error) { + return &array.MetroFracturedResponse{IsFractured: true, State: "Promoted"}, false, nil + } + createOrUpdateJournalEntryFunc = func(_ context.Context, _ string, _ array.VolumeHandle, _ string, _ string, _ string, _ []byte) error { + return nil + } - res, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ - VolumeId: validBlockVolumeID, + res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: validMetroVolumeHandle, + PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "mount", "single-writer", "ext4"), }) + gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(res).To(gomega.Equal(&csi.NodeUnstageVolumeResponse{})) + gomega.Expect(res).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) }) - ginkgo.It("should fail, no targetPath [iSCSI]", func() { - mountInfo := []gofsutil.Info{ - { - Device: validDevName, - Path: stagingPath, - }, - } - fsMock.On("GetUtil").Return(utilMock) - fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) - fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) + ginkgo.It("fails when both arrays are unreachable", func() { + // ensure checking metro state will fail + clientMock.On("GetVolume", mock.Anything, validBaseVolumeID).Return(gopowerstore.Volume{}, errors.New("source array offline")) + clientMock.On("GetVolume", mock.Anything, validRemoteBaseVolumeID).Return(gopowerstore.Volume{}, errors.New("target array offline")) - utilMock.On("Unmount", mock.Anything, stagingPath).Return(nil) + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true + } + defer func() { isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc }() - fsMock.On("Remove", stagingPath).Return(nil) - fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0o640)).Return(nil) + setDefaultClientMocks() + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) + scsiStageVolumeOK(utilMock, fsMock) + // RemoteVolume get fails. So remoteVolume stage fails. + clientMock.On("GetVolume", mock.Anything, "00000000-0000-0000-0000-000000000002").Return(gopowerstore.Volume{}, errors.New("some-error")) + checkMetroStateFunc = func(_ context.Context, _ array.VolumeHandle, _ gopowerstore.Client, _ gopowerstore.Client) (*array.MetroFracturedResponse, bool, error) { + return nil, false, fmt.Errorf("failed to get metro session info") + } + + res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: validMetroVolumeHandle, + PublishContext: getValidPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "mount", "single-writer", "ext4"), + }) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + }) + + ginkgo.It("creates a journal entry when staging local fails", func() { + // when checking metro state, indicate it is fractured and local is demoted + clientMock.On("GetVolume", mock.Anything, validBaseVolumeID).Return(gopowerstore.Volume{ + ID: validBaseVolumeID, + MetroReplicationSessionID: validMetroSessionID, + }, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, validMetroSessionID).Return(gopowerstore.ReplicationSession{ + State: "Fractured", + LocalResourceState: "Demoted", + }, nil) + + clientMock.On("GetVolume", mock.Anything, validRemoteBaseVolumeID).Return(gopowerstore.Volume{ + ID: validRemoteBaseVolumeID, + MetroReplicationSessionID: validMetroSessionID, + }, nil) + + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true + } + defer func() { isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc }() + + createOrUpdateJournalEntryFunc = func(_ context.Context, _ string, _ array.VolumeHandle, _ string, _ string, _ string, _ []byte) error { + return nil + } + + setDefaultClientMocks() + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) + scsiStageVolumeFail(utilMock, fsMock) + scsiStageRemoteMetroVolumeOK(utilMock, fsMock) + + res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: validMetroVolumeHandle, + PublishContext: getValidUniformMetroPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "mount", "single-writer", "ext4"), + }) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) + }) + + ginkgo.It("creates a journal entry when staging remote fails", func() { + // when checking metro state, indicate it is fractured and local is promoted + clientMock.On("GetVolume", mock.Anything, validBaseVolumeID).Return(gopowerstore.Volume{ + ID: validBaseVolumeID, + MetroReplicationSessionID: validMetroSessionID, + }, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, validMetroSessionID).Return(gopowerstore.ReplicationSession{ + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + + clientMock.On("GetVolume", mock.Anything, validRemoteBaseVolumeID).Return(gopowerstore.Volume{ + ID: validRemoteBaseVolumeID, + MetroReplicationSessionID: validMetroSessionID, + }, nil) + + createOrUpdateJournalEntryFunc = func(_ context.Context, _ string, _ array.VolumeHandle, _ string, _ string, _ string, _ []byte) error { + return nil + } + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true + } + defer func() { isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc }() + + setDefaultClientMocks() + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) + // successfully stage the local + scsiStageVolumeOK(utilMock, fsMock) + // fail staging the remote + scsiStageVolumeFail(utilMock, fsMock) + + res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: validMetroVolumeHandle, + PublishContext: getValidUniformMetroPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "mount", "single-writer", "ext4"), + }) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) + }) + + ginkgo.It("fails to create journal entry", func() { + // stage the local side, should fail on the remote, triggering the creation of the journal entry + clientMock.On("GetVolume", mock.Anything, validBaseVolumeID).Return(gopowerstore.Volume{ + ID: validBaseVolumeID, + MetroReplicationSessionID: validMetroSessionID, + }, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, validMetroSessionID).Return(gopowerstore.ReplicationSession{ + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + + clientMock.On("GetVolume", mock.Anything, validRemoteBaseVolumeID).Return(gopowerstore.Volume{ + ID: validRemoteBaseVolumeID, + MetroReplicationSessionID: validMetroSessionID, + }, nil) + + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true + } + defer func() { isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc }() + + setDefaultClientMocks() + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) + scsiStageVolumeOK(utilMock, fsMock) + scsiStageVolumeFail(utilMock, fsMock) + + createOrUpdateJournalEntryFunc = func(_ context.Context, _ string, _ array.VolumeHandle, _ string, _ string, _ string, _ []byte) error { + return fmt.Errorf("unable to create journal entry") + } + + res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: validMetroVolumeHandle, + PublishContext: getValidUniformMetroPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "mount", "single-writer", "ext4"), + }) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + }) + }) + }) + + ginkgo.Describe("calling NodeUnstage()", func() { + stagingPath := filepath.Join(nodeStagePrivateDir, validBaseVolumeID) + + ginkgo.When("unstaging block volume", func() { + ginkgo.It("should succeed [iSCSI]", func() { + mountInfo := []gofsutil.Info{ + { + Device: validDevName, + Path: stagingPath, + }, + } + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) + fsMock.On("GetUtil").Return(utilMock) + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) + fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) + utilMock.On("Unmount", mock.Anything, stagingPath).Return(nil) + + fsMock.On("Remove", stagingPath).Return(nil) + fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0o640)).Return(nil) + + iscsiConnectorMock.On("DisconnectVolumeByDeviceName", mock.Anything, validDevName).Return(nil) + + fsMock.On("Remove", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID)).Return(nil) + fsMock.On("IsNotExist", mock.Anything).Return(false) + + res, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: validBlockVolumeHandle, + StagingTargetPath: nodeStagePrivateDir, + }) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeUnstageVolumeResponse{})) + }) + ginkgo.It("should fail, no targetPath [iSCSI]", func() { + mountInfo := []gofsutil.Info{ + { + Device: validDevName, + Path: stagingPath, + }, + } + + fsMock.On("GetUtil").Return(utilMock) + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) + fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) + utilMock.On("Unmount", mock.Anything, stagingPath).Return(nil) + + fsMock.On("Remove", stagingPath).Return(nil) + fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0o640)).Return(nil) iscsiConnectorMock.On("DisconnectVolumeByDeviceName", mock.Anything, validDevName).Return(nil) @@ -1889,7 +2398,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("IsNotExist", mock.Anything).Return(false) _, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, StagingTargetPath: "", }) gomega.Expect(err.Error()).To(gomega.ContainSubstring("staging target path is required")) @@ -1899,6 +2408,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { VolumeId: invalidBlockVolumeID, StagingTargetPath: nodeStagePrivateDir, }) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't find array with ID")) }) ginkgo.It("should fail, because no mounts [iSCSI]", func() { @@ -1908,7 +2418,13 @@ var _ = ginkgo.Describe("CSINodeService", func() { Path: stagingPath, }, } - + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, errors.New("fail")) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) @@ -1924,7 +2440,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("IsNotExist", mock.Anything).Return(false) _, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, StagingTargetPath: nodeStagePrivateDir, }) gomega.Expect(err.Error()).To(gomega.ContainSubstring("could not reliably determine existing mount for path")) @@ -1936,7 +2452,13 @@ var _ = ginkgo.Describe("CSINodeService", func() { Path: stagingPath, }, } - + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) @@ -1952,7 +2474,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("IsNotExist", mock.Anything).Return(false) _, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, StagingTargetPath: nodeStagePrivateDir, }) gomega.Expect(err.Error()).To(gomega.ContainSubstring("could not unmount de")) @@ -1964,7 +2486,12 @@ var _ = ginkgo.Describe("CSINodeService", func() { Path: "invalid", }, } - + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", mock.Anything).Return([]byte{}, nil) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) @@ -1980,7 +2507,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("IsNotExist", mock.Anything).Return(false) res, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, StagingTargetPath: nodeStagePrivateDir, }) gomega.Expect(err).To(gomega.BeNil()) @@ -1991,7 +2518,12 @@ var _ = ginkgo.Describe("CSINodeService", func() { mountInfo := []gofsutil.Info{{Device: validDevName, Path: stagingPath}} remoteStagingPath := filepath.Join(nodeStagePrivateDir, validRemoteVolID) remoteMountInfo := []gofsutil.Info{{Device: validDevName, Path: remoteStagingPath}} - + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(4) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil).Once() @@ -2010,7 +2542,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("Remove", path.Join(nodeSvc.opts.TmpDir, validRemoteVolID)).Return(nil).Once() fsMock.On("IsNotExist", mock.Anything).Return(false) - metroVolumeID := fmt.Sprintf("%s:%s/%s", validBlockVolumeID, validRemoteVolID, secondValidIP) + metroVolumeID := fmt.Sprintf("%s:%s/%s", validBlockVolumeHandle, validRemoteVolID, secondValidIP) res, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ VolumeId: metroVolumeID, StagingTargetPath: nodeStagePrivateDir, @@ -2028,6 +2560,14 @@ var _ = ginkgo.Describe("CSINodeService", func() { }, } + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + Wwn: "naa.68ccf09800e23ab798312a05426acae0", + }, nil) + fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) @@ -2036,14 +2576,15 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("Remove", stagingPath).Return(nil) fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0o640)).Return(nil) - + fcConnectorMock.On("DisconnectVolumeByWWN", mock.Anything, validDeviceWWN).Return(errors.New("mock disconnect failure")).Once() + fcConnectorMock.On("DisconnectVolumeByWWN", mock.Anything, validDeviceWWN).Return(nil) fcConnectorMock.On("DisconnectVolumeByDeviceName", mock.Anything, validDevName).Return(nil) fsMock.On("Remove", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID)).Return(nil) fsMock.On("IsNotExist", mock.Anything).Return(false) res, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, StagingTargetPath: nodeStagePrivateDir, }) gomega.Expect(err).To(gomega.BeNil()) @@ -2058,7 +2599,12 @@ var _ = ginkgo.Describe("CSINodeService", func() { Path: stagingPath, }, } - + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) @@ -2074,7 +2620,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("IsNotExist", mock.Anything).Return(false) res, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, StagingTargetPath: nodeStagePrivateDir, }) gomega.Expect(err).To(gomega.BeNil()) @@ -2091,6 +2637,12 @@ var _ = ginkgo.Describe("CSINodeService", func() { Path: remnantStagingPath, }, } + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(4) @@ -2111,12 +2663,58 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("Remove", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID)).Return(nil) res, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, + StagingTargetPath: nodeStagePrivateDir, + }) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeUnstageVolumeResponse{})) + }) + ginkgo.It("should succeed when volume has already been deleted on array [iSCSI]", func() { + mountInfo := []gofsutil.Info{ + { + Device: validDevName, + Path: stagingPath, + }, + } + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, + }, + }) + fsMock.On("GetUtil").Return(utilMock) + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) + fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) + utilMock.On("Unmount", mock.Anything, stagingPath).Return(nil) + + fsMock.On("Remove", stagingPath).Return(nil) + fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0o640)).Return(nil) + + iscsiConnectorMock.On("DisconnectVolumeByDeviceName", mock.Anything, validDevName).Return(nil) + + fsMock.On("Remove", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID)).Return(nil) + fsMock.On("IsNotExist", mock.Anything).Return(false) + + res, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: validBlockVolumeHandle, StagingTargetPath: nodeStagePrivateDir, }) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(res).To(gomega.Equal(&csi.NodeUnstageVolumeResponse{})) }) + ginkgo.It("should failed due to inability to get volume [iSCSI]", func() { + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusBadGateway, + }, + }) + + _, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: validBlockVolumeHandle, + StagingTargetPath: nodeStagePrivateDir, + }) + gomega.Expect(err).ToNot(gomega.BeNil()) + }) }) ginkgo.When("unstaging nfs volume", func() { ginkgo.It("should succeed", func() { @@ -2126,7 +2724,12 @@ var _ = ginkgo.Describe("CSINodeService", func() { Path: stagingPath, }, } - + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) @@ -2160,7 +2763,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("Mount", mock.Anything, stagingPath, validTargetPath, "").Return(nil) res, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2183,7 +2786,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("Mount", mock.Anything, stagingPath, validTargetPath, "").Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2205,7 +2808,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("Mount", mock.Anything, stagingPath, validTargetPath, "ext4", "ro").Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2227,7 +2830,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("Mount", mock.Anything, stagingPath, validTargetPath, "").Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2249,7 +2852,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("Mount", mock.Anything, stagingPath, validTargetPath, "").Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2271,7 +2874,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("Mount", mock.Anything, stagingPath, validTargetPath, "ext4").Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2293,7 +2896,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("Mount", mock.Anything, stagingPath, validTargetPath, "").Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2304,19 +2907,28 @@ var _ = ginkgo.Describe("CSINodeService", func() { }) }) ginkgo.When("publishing block volume as mount with multi-writer", func() { - ginkgo.It("should fail", func() { + ginkgo.It("should succeed", func() { fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) + fsMock.On("GetUtil").Return(utilMock) + fsMock.On("MkFileIdempotent", validTargetPath).Return(true, nil) + utilMock.On("BindMount", mock.Anything, stagingPath, validTargetPath).Return(nil) + fsMock.On("MkdirAll", validTargetPath, mock.Anything).Return(nil) + utilMock.On("GetDiskFormat", mock.Anything, stagingPath).Return("", nil) + fsMock.On("ExecCommand", "mkfs.ext4", "-E", "nodiscard", "-F", stagingPath).Return([]byte{}, nil) + utilMock.On("Mount", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + utilMock.On("Unmount", mock.Anything, mock.Anything).Return(nil) + fsMock.On("RemoveAll", mock.Anything).Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "multiple-writer", ""), Readonly: false, }) - gomega.Expect(err.Error()).To(gomega.ContainSubstring("Mount volumes do not support AccessMode MULTI_NODE_MULTI_WRITER")) + gomega.Expect(err).To(gomega.BeNil()) }) }) ginkgo.When("publishing block volume as raw block", func() { @@ -2329,7 +2941,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("BindMount", mock.Anything, stagingPath, validTargetPath).Return(nil) res, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2350,7 +2962,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("BindMount", mock.Anything, stagingPath, validTargetPath, "ro").Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2370,7 +2982,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("BindMount", mock.Anything, stagingPath, validTargetPath).Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2390,7 +3002,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("BindMount", mock.Anything, stagingPath, validTargetPath).Return(errors.New("failed to bind")) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2405,7 +3017,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, errors.New("fail")) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2428,7 +3040,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2478,7 +3090,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { ginkgo.When("No target path specified", func() { ginkgo.It("should fail", func() { res, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: "", @@ -2492,7 +3104,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { ginkgo.When("Invalid volume capabilities specified", func() { ginkgo.It("should fail", func() { res, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2506,7 +3118,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { ginkgo.When("No staging target path specified", func() { ginkgo.It("should fail", func() { res, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: "", TargetPath: validTargetPath, @@ -2605,7 +3217,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("Remove", mock.Anything).Return(nil) res, err := nodeSvc.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, TargetPath: validTargetPath, }) gomega.Expect(err).To(gomega.BeNil()) @@ -2640,7 +3252,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { ginkgo.When("No target path specified", func() { ginkgo.It("should fail", func() { res, err := nodeSvc.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, TargetPath: "", }) gomega.Expect(err.Error()).To(gomega.Equal("rpc error: code = InvalidArgument desc = target path required")) @@ -2664,7 +3276,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(nil, errors.New("error")) fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, os.ErrNotExist) res, err := nodeSvc.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, TargetPath: validTargetPath, }) gomega.Expect(err.Error()).To(gomega.ContainSubstring("could not reliably determine existing mount status")) @@ -2687,7 +3299,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, os.ErrNotExist) utilMock.On("Unmount", mock.Anything, validTargetPath).Return(errors.New("Unmount failed")) res, err := nodeSvc.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, TargetPath: validTargetPath, }) gomega.Expect(err.Error()).To(gomega.ContainSubstring("could not unmount dev")) @@ -2703,7 +3315,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -2716,7 +3328,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("FindFSType", mock.Anything, mock.Anything).Return("ext4", nil) fsMock.On("ExecCommandOutput", mock.Anything, mock.Anything, mock.Anything).Return([]byte("version 5.0.0"), nil) utilMock.On("ResizeFS", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) gomega.Ω(err).To(gomega.BeNil()) gomega.Ω(res).To(gomega.Equal(&csi.NodeExpandVolumeResponse{})) }) @@ -2724,7 +3336,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "tn1-csivol-123456", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -2738,7 +3350,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("ExecCommandOutput", mock.Anything, mock.Anything, mock.Anything).Return([]byte("version 5.0.0"), nil) utilMock.On("ResizeFS", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) os.Setenv("X_CSM_AUTH_ENABLED", "true") - res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) gomega.Ω(err).To(gomega.BeNil()) gomega.Ω(res).To(gomega.Equal(&csi.NodeExpandVolumeResponse{})) }) @@ -2747,7 +3359,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -2760,13 +3372,13 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("FindFSType", mock.Anything, mock.Anything).Return("xfs", nil) fsMock.On("ExecCommandOutput", mock.Anything, mock.Anything, mock.Anything).Return([]byte("version 5.0.0"), nil) utilMock.On("ResizeFS", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) gomega.Ω(err).To(gomega.BeNil()) gomega.Ω(res).To(gomega.Equal(&csi.NodeExpandVolumeResponse{})) }) ginkgo.It("should succeed [metro-volumes]", func() { fsMock.On("GetUtil").Return(utilMock) - metroVolumeID := fmt.Sprintf("%s:%s/%s", validBlockVolumeID, validRemoteVolID, secondValidIP) + metroVolumeID := fmt.Sprintf("%s:%s/%s", validBlockVolumeHandle, validRemoteVolID, secondValidIP) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", ID: metroVolumeID, @@ -2797,7 +3409,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -2810,7 +3422,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("FindFSType", mock.Anything, mock.Anything).Return("xfs", nil) fsMock.On("ExecCommandOutput", mock.Anything, mock.Anything, mock.Anything).Return([]byte("version 5.0.0"), nil) utilMock.On("ResizeFS", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("resize Failed ext4")) - res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) gomega.Ω(err.Error()).To(gomega.ContainSubstring("resize Failed ext4")) gomega.Ω(res).To(gomega.BeNil()) }) @@ -2818,7 +3430,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -2832,7 +3444,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("FindFSType", mock.Anything, mock.Anything).Return("ext4", nil) fsMock.On("ExecCommandOutput", mock.Anything, mock.Anything, mock.Anything).Return([]byte("version 5.0.0"), nil) utilMock.On("ResizeFS", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("resize Failed xfs")) - res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) gomega.Ω(err.Error()).To(gomega.ContainSubstring("resize Failed xfs")) gomega.Ω(res).To(gomega.BeNil()) }) @@ -2846,7 +3458,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -2870,7 +3482,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("FindFSType", mock.Anything, mock.Anything).Return("ext4", nil) fsMock.On("ExecCommandOutput", mock.Anything, mock.Anything, mock.Anything).Return([]byte("version 5.0.0"), nil) utilMock.On("ResizeFS", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) gomega.Ω(err).To(gomega.BeNil()) gomega.Ω(res).To(gomega.Equal(&csi.NodeExpandVolumeResponse{})) }) @@ -2880,7 +3492,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -2905,7 +3517,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("FindFSType", mock.Anything, mock.Anything).Return("xfs", nil) fsMock.On("ExecCommandOutput", mock.Anything, mock.Anything, mock.Anything).Return([]byte("version 5.0.0"), nil) utilMock.On("ResizeFS", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) gomega.Ω(err).To(gomega.BeNil()) gomega.Ω(res).To(gomega.Equal(&csi.NodeExpandVolumeResponse{})) }) @@ -2916,7 +3528,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, Wwn: "naa.6090a038f0cd4e5bdaa8248e6856d4fe:3", @@ -2930,7 +3542,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("DeviceRescan", mock.Anything, mock.Anything).Return(nil) utilMock.On("GetMpathNameFromDevice", mock.Anything, mock.Anything).Return("mpatha", nil) utilMock.On("ResizeMultipath", mock.Anything, mock.Anything).Return(nil) - res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, true)) + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, true)) gomega.Ω(err).To(gomega.BeNil()) gomega.Ω(res).To(gomega.Equal(&csi.NodeExpandVolumeResponse{})) }) @@ -2962,14 +3574,14 @@ var _ = ginkgo.Describe("CSINodeService", func() { clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, Wwn: "naa.6090a038f0cd4e5bdaa8248e6856d4fe:3", }, nil) _, err := nodeSvc.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, VolumePath: "", CapacityRange: &csi.CapacityRange{ RequiredBytes: 2234234, @@ -2985,14 +3597,14 @@ var _ = ginkgo.Describe("CSINodeService", func() { clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, Wwn: "naa.6090a038f0cd4e5bdaa8248e6856d4fe:3", }, errors.New("err")).Times(1) _, err := nodeSvc.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath, CapacityRange: &csi.CapacityRange{ RequiredBytes: 2234234, @@ -3007,7 +3619,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -3017,7 +3629,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { MountPoint: stagingPath, }, errors.New("offline")).Times(1) fsMock.On("MkdirAll", mock.Anything, mock.Anything).Return(errors.New("Unable to create dirs")) - _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) + _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) gomega.Ω(err.Error()).To(gomega.ContainSubstring("Failed to find mount info for")) }) }) @@ -3026,7 +3638,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -3037,7 +3649,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { }, errors.New("offline")).Times(1) fsMock.On("MkdirAll", mock.Anything, mock.Anything).Return(nil) utilMock.On("Mount", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("bad mount")) - _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) + _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) gomega.Ω(err.Error()).To(gomega.ContainSubstring("Failed to find mount info for")) }) }) @@ -3046,7 +3658,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -3071,7 +3683,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("FindFSType", mock.Anything, mock.Anything).Return("xfs", nil) fsMock.On("ExecCommandOutput", mock.Anything, mock.Anything, mock.Anything).Return([]byte("version 5.0.0"), nil) utilMock.On("ResizeFS", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) gomega.Ω(err).To(gomega.BeNil()) gomega.Ω(res).To(gomega.Equal(&csi.NodeExpandVolumeResponse{})) }) @@ -3081,7 +3693,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -3101,7 +3713,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { MountPoint: stagingPath, }, errors.New("again")).Times(1) - _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) + _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) gomega.Ω(err.Error()).To(gomega.ContainSubstring("Failed to find mount info for")) }) }) @@ -3110,7 +3722,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -3130,7 +3742,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { MountPoint: stagingPath, }, nil).Times(1) utilMock.On("DeviceRescan", mock.Anything, mock.Anything).Return(errors.New("Failed to rescan device")) - _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) + _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) gomega.Ω(err.Error()).To(gomega.ContainSubstring("Failed to rescan device")) }) }) @@ -3139,7 +3751,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -3160,7 +3772,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { }, nil).Times(1) utilMock.On("DeviceRescan", mock.Anything, mock.Anything).Return(nil) utilMock.On("ResizeMultipath", mock.Anything, mock.Anything).Return(errors.New("mpath resize error")) - _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) + _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) gomega.Ω(err.Error()).To(gomega.ContainSubstring("mpath resize error")) }) }) @@ -3207,7 +3819,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("ExecCommand", "mkfs.ext4", "-E", "nodiscard", "-F", mock.Anything).Return([]byte{}, nil) utilMock.On("Mount", mock.Anything, mock.Anything, mock.Anything, "ext4").Return(nil) res, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -3247,7 +3859,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { }, }, nil) ctrlMock.On("ControllerPublishVolume", mock.Anything, &csi.ControllerPublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, NodeId: validNodeID, VolumeContext: map[string]string{ identifiers.KeyArrayID: firstValidIP, @@ -3281,7 +3893,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("Remove", mock.Anything).Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -3346,7 +3958,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { utilMock.On("Unmount", mock.Anything, mock.Anything).Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -3365,7 +3977,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, nil) fsMock.On("MkdirAll", mock.Anything, mock.Anything).Return(nil).Times(2) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -3384,7 +3996,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, os.ErrNotExist) fsMock.On("MkdirAll", mock.Anything, mock.Anything).Return(errors.New("err")).Times(2) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -3414,7 +4026,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { }, }, errors.New("Failed")) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -3444,7 +4056,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { }, }, nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -3474,7 +4086,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { }, }, nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -3529,7 +4141,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("ExecCommand", "mkfs.xfs", "-K", mock.Anything, "-m", mock.Anything).Return([]byte{}, nil) utilMock.On("Mount", mock.Anything, mock.Anything, mock.Anything, "xfs", mock.Anything).Return(errors.New("err")) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -3553,7 +4165,13 @@ var _ = ginkgo.Describe("CSINodeService", func() { Path: validTargetPath, }, } - fsMock.On("ReadFile", ephemerallockfile).Return([]byte(validBlockVolumeID), nil) + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) + fsMock.On("ReadFile", ephemerallockfile).Return([]byte(validBlockVolumeHandle), nil) fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, nil) fsMock.On("GetUtil").Return(utilMock) @@ -3571,14 +4189,14 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("IsNotExist", mock.Anything).Return(false) fsMock.On("ReadFile", mock.Anything).Return([]byte("Some data"), nil) ctrlMock.On("ControllerUnpublishVolume", mock.Anything, &csi.ControllerUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, NodeId: validNodeID, }).Return(&csi.ControllerUnpublishVolumeResponse{}, nil) ctrlMock.On("DeleteVolume", mock.Anything, &csi.DeleteVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, }).Return(&csi.DeleteVolumeResponse{}, nil) res, err := nodeSvc.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, TargetPath: validTargetPath, }) gomega.Ω(err).To(gomega.BeNil()) @@ -3600,9 +4218,9 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, nil) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil) fsMock.On("GetUtil").Return(utilMock) - fsMock.On("ReadFile", ephemerallockfile).Return([]byte(validBlockVolumeID), os.ErrNotExist) + fsMock.On("ReadFile", ephemerallockfile).Return([]byte(validBlockVolumeHandle), os.ErrNotExist) _, err := nodeSvc.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, TargetPath: validTargetPath, }) gomega.Ω(err.Error()).To(gomega.ContainSubstring("Was unable to read lockfile")) @@ -3616,7 +4234,13 @@ var _ = ginkgo.Describe("CSINodeService", func() { Path: validTargetPath, }, } - fsMock.On("ReadFile", ephemerallockfile).Return([]byte(validBlockVolumeID), nil) + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) + fsMock.On("ReadFile", ephemerallockfile).Return([]byte(validBlockVolumeHandle), nil) fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, nil) fsMock.On("GetUtil").Return(utilMock) @@ -3634,14 +4258,14 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("IsNotExist", mock.Anything).Return(false) fsMock.On("ReadFile", mock.Anything).Return([]byte("Some data"), nil) ctrlMock.On("ControllerUnpublishVolume", mock.Anything, &csi.ControllerUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, NodeId: validNodeID, }).Return(&csi.ControllerUnpublishVolumeResponse{}, errors.New("failed")) ctrlMock.On("DeleteVolume", mock.Anything, &csi.DeleteVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, }).Return(&csi.DeleteVolumeResponse{}, nil) _, err := nodeSvc.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, TargetPath: validTargetPath, }) gomega.Ω(err.Error()).To(gomega.ContainSubstring("Inline ephemeral controller unpublish")) @@ -3655,9 +4279,15 @@ var _ = ginkgo.Describe("CSINodeService", func() { Path: validTargetPath, }, } - fsMock.On("ReadFile", ephemerallockfile).Return([]byte(validBlockVolumeID), nil) - fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, nil) - + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) + fsMock.On("ReadFile", ephemerallockfile).Return([]byte(validBlockVolumeHandle), nil) + fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, nil) + fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) @@ -3673,14 +4303,14 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("IsNotExist", mock.Anything).Return(false) fsMock.On("ReadFile", mock.Anything).Return([]byte("Some data"), nil) ctrlMock.On("ControllerUnpublishVolume", mock.Anything, &csi.ControllerUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, NodeId: validNodeID, }).Return(&csi.ControllerUnpublishVolumeResponse{}, nil) ctrlMock.On("DeleteVolume", mock.Anything, &csi.DeleteVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, }).Return(&csi.DeleteVolumeResponse{}, errors.New("failed")) _, err := nodeSvc.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, TargetPath: validTargetPath, }) gomega.Ω(err.Error()).To(gomega.ContainSubstring("failed")) @@ -3823,10 +4453,8 @@ var _ = ginkgo.Describe("CSINodeService", func() { conn, nil, ) - nodeLabelsRetrieverMock.On("BuildConfigFromFlags", mock.Anything, mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("GetNodeLabels", mock.Anything, mock.Anything).Return(map[string]string{"max-powerstore-volumes-per-node": "2"}, nil) - nodeLabelsRetrieverMock.On("InClusterConfig", mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("NewForConfig", mock.Anything).Return(nil, nil) + + k8sutils.Kubeclient.SetNodeLabel(context.Background(), nodeSvc.opts.KubeNodeName, "max-powerstore-volumes-per-node", "2") res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) gomega.Expect(err).To(gomega.BeNil()) @@ -3864,10 +4492,6 @@ var _ = ginkgo.Describe("CSINodeService", func() { conn, nil, ) - nodeLabelsRetrieverMock.On("BuildConfigFromFlags", mock.Anything, mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("GetNodeLabels", mock.Anything, mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("InClusterConfig", mock.Anything).Return(nil, errors.New("Unable to create kubeclientset")) - nodeLabelsRetrieverMock.On("NewForConfig", mock.Anything).Return(nil, nil) res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) gomega.Expect(err).To(gomega.BeNil()) @@ -3885,6 +4509,63 @@ var _ = ginkgo.Describe("CSINodeService", func() { }) }) + ginkgo.When("calling NodeGetInfo with metro and match labels for zone1", func() { + ginkgo.It("should return correct NodeGetInfoResponse", func() { + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn2"}, + }, + }, nil) + conn, _ := net.Dial("udp", "127.0.0.1:80") + fsMock.On("NetDial", mock.Anything).Return( + conn, + nil, + ) + + nodeSvc.SetArrays(getMetroTestArrays()) + res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.AccessibleTopology.Segments).To(gomega.HaveKeyWithValue("topology.kubernetes.io/zone", "zone1")) + }) + }) + ginkgo.When("calling NodeGetInfo with metro and match labels for zone2", func() { + ginkgo.It("should return correct NodeGetInfo response", func() { + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn2"}, + }, + }, nil) + conn, _ := net.Dial("udp", "127.0.0.1:80") + fsMock.On("NetDial", mock.Anything).Return( + conn, + nil, + ) + nodeSvc.SetArrays(getMetroTestArrays()) + + k8sutils.Kubeclient.SetNodeLabel(context.Background(), nodeSvc.opts.KubeNodeName, "topology.kubernetes.io/zone", "zone2") + + res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.AccessibleTopology.Segments).To(gomega.HaveKeyWithValue("topology.kubernetes.io/zone", "zone2")) + }) + }) + ginkgo.When("MaxVolumesPerNode is set via environment variable at the time of installation", func() { ginkgo.It("should return correct MaxVolumesPerNode in response", func() { clientMock.On("GetNASServers", mock.Anything). @@ -4079,117 +4760,6 @@ var _ = ginkgo.Describe("CSINodeService", func() { })) }) - ginkgo.When("reusing host", func() { - ginkgo.It("should properly deal with additional IPs", func() { - nodeSvc.useFC[firstGlobalID] = true - nodeID := nodeSvc.nodeID - nodeSvc.nodeID = nodeID + "-" + "192.168.0.1" - nodeSvc.reusedHost = true - clientMock.On("GetNASServers", mock.Anything). - Return(nasData, nil) - conn, _ := net.Dial("udp", "127.0.0.1:80") - fsMock.On("NetDial", mock.Anything).Return( - conn, - nil, - ) - clientMock.On("GetHostByName", mock.Anything, nodeID). - Return(gopowerstore.Host{ - ID: "host-id", - Initiators: []gopowerstore.InitiatorInstance{ - { - ActiveSessions: []gopowerstore.ActiveSessionInstance{ - { - PortName: validFCTargetsWWPN[0], - }, - }, - PortName: validFCTargetsWWPN[0], - PortType: gopowerstore.InitiatorProtocolTypeEnumFC, - }, - { - ActiveSessions: []gopowerstore.ActiveSessionInstance{ - { - PortName: validFCTargetsWWPN[1], - }, - }, - PortName: validFCTargetsWWPN[1], - PortType: gopowerstore.InitiatorProtocolTypeEnumFC, - }, - }, - Name: "host-name", - }, nil) - setDefaultNodeLabelsMock() - - res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ - NodeId: nodeSvc.nodeID, - AccessibleTopology: &csi.Topology{ - Segments: map[string]string{ - identifiers.Name + "/" + firstValidIP + "-nfs": "true", - identifiers.Name + "/" + firstValidIP + "-fc": "true", - identifiers.Name + "/" + secondValidIP + "-nfs": "true", - }, - }, - MaxVolumesPerNode: 0, - })) - }) - - ginkgo.When("there is no ip in nodeID", func() { - ginkgo.It("should not return FC topology key", func() { - nodeSvc.useFC[firstGlobalID] = true - nodeID := nodeSvc.nodeID - nodeSvc.nodeID = "nodeid-with-no-ip" - nodeSvc.reusedHost = true - conn, _ := net.Dial("udp", "127.0.0.1:80") - clientMock.On("GetNASServers", mock.Anything). - Return(nasData, nil) - fsMock.On("NetDial", mock.Anything).Return( - conn, - nil, - ) - clientMock.On("GetHostByName", mock.Anything, nodeID). - Return(gopowerstore.Host{ - ID: "host-id", - Initiators: []gopowerstore.InitiatorInstance{ - { - ActiveSessions: []gopowerstore.ActiveSessionInstance{ - { - PortName: validFCTargetsWWPN[0], - }, - }, - PortName: validFCTargetsWWPN[0], - PortType: gopowerstore.InitiatorProtocolTypeEnumFC, - }, - { - ActiveSessions: []gopowerstore.ActiveSessionInstance{ - { - PortName: validFCTargetsWWPN[1], - }, - }, - PortName: validFCTargetsWWPN[1], - PortType: gopowerstore.InitiatorProtocolTypeEnumFC, - }, - }, - Name: "host-name", - }, nil) - setDefaultNodeLabelsMock() - - res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ - NodeId: nodeSvc.nodeID, - AccessibleTopology: &csi.Topology{ - Segments: map[string]string{ - identifiers.Name + "/" + firstValidIP + "-nfs": "true", - identifiers.Name + "/" + secondValidIP + "-nfs": "true", - }, - }, - MaxVolumesPerNode: 0, - })) - }) - }) - }) - ginkgo.When("we can not get info about hosts from array", func() { ginkgo.It("should not return FC topology key", func() { nodeSvc.useFC[firstGlobalID] = true @@ -4424,6 +4994,136 @@ var _ = ginkgo.Describe("CSINodeService", func() { })) }) + ginkgo.When("using iSCSI on multiple networks", func() { + ginkgo.BeforeEach(func() { + options = []variableOption{withMockNumberOfISCSITargets(4)} + }) + ginkgo.AfterEach(func() { + options = []variableOption{} + }) + ginkgo.It("should return iSCSI topology segments", func() { + goiscsi.GOISCSIMock.InduceDiscoveryError = false + nodeSvc.useNVME[firstGlobalID] = false + nodeSvc.useNFS = false + clientMock.On("GetNASServers", mock.Anything). + Return([]gopowerstore.NAS{}, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW1", + }, + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW1", + }, + { + Address: "192.168.2.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW2", + }, + { + Address: "192.168.2.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW2", + }, + }, nil) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{ + Name: validClusterName, + NVMeNQN: validNVMEInitiators[0], + }, nil) + conn, _ := net.Dial("udp", "127.0.0.1:80") + fsMock.On("NetDial", mock.Anything).Return( + conn, + nil, + ) + setDefaultNodeLabelsMock() + + res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ + NodeId: nodeSvc.nodeID, + AccessibleTopology: &csi.Topology{ + Segments: map[string]string{ + identifiers.Name + "/" + firstValidIP + "-iscsi": "true", + }, + }, + MaxVolumesPerNode: 0, + })) + }) + }) + + ginkgo.When("using NVMeTCP on multiple networks", func() { + ginkgo.BeforeEach(func() { + options = []variableOption{withMockNumberOfNVMeTCPTargets(2)} + }) + ginkgo.AfterEach(func() { + options = []variableOption{} + }) + ginkgo.It("should return NVMeTCP topology segments", func() { + nodeSvc.useNVME[firstGlobalID] = true + nodeSvc.useFC[firstGlobalID] = false + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + }, nil) + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW1", + }, + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW1", + }, + { + Address: "192.168.2.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW2", + }, + { + Address: "192.168.2.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW2", + }, + }, nil) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{ + Name: validClusterName, + NVMeNQN: validNVMEInitiators[0], + }, nil) + conn, _ := net.Dial("udp", "127.0.0.1:80") + fsMock.On("NetDial", mock.Anything).Return( + conn, + nil, + ) + setDefaultNodeLabelsMock() + + res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ + NodeId: nodeSvc.nodeID, + AccessibleTopology: &csi.Topology{ + Segments: map[string]string{ + identifiers.Name + "/" + firstValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-nvmetcp": "true", + identifiers.Name + "/" + secondValidIP + "-nfs": "true", + }, + }, + MaxVolumesPerNode: 0, + })) + }) + }) + ginkgo.When("target can not be discovered", func() { ginkgo.It("should not return nvme topology key", func() { goiscsi.GOISCSIMock.InduceDiscoveryError = true @@ -4551,6 +5251,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { }}, Name: "host-name", }}, nil) + // clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) @@ -4794,7 +5495,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { ginkgo.When("volume path is missing", func() { ginkgo.It("should fail", func() { - req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeID, VolumePath: ""} + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeHandle, VolumePath: ""} res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) gomega.Expect(res).To(gomega.BeNil()) @@ -4821,7 +5522,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusBadRequest}, }) - req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeID, VolumePath: validTargetPath} + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath} res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) gomega.Expect(res).To(gomega.BeNil()) @@ -4837,7 +5538,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusBadRequest}, }) - req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeID, VolumePath: validTargetPath} + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath} res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) gomega.Expect(res).To(gomega.BeNil()) @@ -4855,7 +5556,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusBadRequest}, }) - req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeID, VolumePath: validTargetPath} + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath} res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) gomega.Expect(res).To(gomega.BeNil()) @@ -4871,7 +5572,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusNotFound}, }) - req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeID, VolumePath: validTargetPath} + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath} res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) gomega.Expect(err).To(gomega.BeNil()) @@ -4894,7 +5595,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusNotFound}, }) - req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeID, VolumePath: validTargetPath} + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath} res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) gomega.Expect(err).To(gomega.BeNil()) @@ -4917,7 +5618,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { clientMock.On("GetHost", mock.Anything, validHostID). Return(gopowerstore.Host{ID: validHostID, Name: validHostName}, nil) - req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeID, VolumePath: validTargetPath} + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath} res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) gomega.Expect(err).To(gomega.BeNil()) @@ -4951,7 +5652,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { }, }, nil) - req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeID, VolumePath: validTargetPath} + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath} res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) gomega.Expect(err).To(gomega.BeNil()) @@ -5126,7 +5827,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, errors.New("fail")) req := &csi.NodeGetVolumeStatsRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath, StagingTargetPath: nodeStagePrivateDir, } @@ -5142,7 +5843,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) req := &csi.NodeGetVolumeStatsRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath, StagingTargetPath: nodeStagePrivateDir, } @@ -5162,7 +5863,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, errors.New("fail")) req := &csi.NodeGetVolumeStatsRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath, StagingTargetPath: "", } @@ -5178,7 +5879,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) req := &csi.NodeGetVolumeStatsRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath, StagingTargetPath: "", } @@ -5204,7 +5905,7 @@ var _ = ginkgo.Describe("CSINodeService", func() { }, nil) req := &csi.NodeGetVolumeStatsRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath, StagingTargetPath: "", } @@ -5254,15 +5955,33 @@ func TestGetNodeOptions(t *testing.T) { } nodeSvc.SetArrays(arrays) nodeSvc.SetDefaultArray(arrays[firstValidIP]) - t.Run("success test", func(_ *testing.T) { - csictx.Setenv(context.Background(), identifiers.EnvNodeIDFilePath, "") - csictx.Setenv(context.Background(), identifiers.EnvNodeNamePrefix, "") - csictx.Setenv(context.Background(), identifiers.EnvKubeNodeName, "") - csictx.Setenv(context.Background(), identifiers.EnvNodeChrootPath, "") - csictx.Setenv(context.Background(), identifiers.EnvTmpDir, "") - csictx.Setenv(context.Background(), identifiers.EnvFCPortsFilterFilePath, "") - csictx.Setenv(context.Background(), identifiers.EnvEnableCHAP, "") - getNodeOptions() + + t.Run("success test with valid maxVolumesPerNode", func(_ *testing.T) { + ctx := context.Background() + csictx.Setenv(ctx, identifiers.EnvNodeIDFilePath, "") + csictx.Setenv(ctx, identifiers.EnvNodeNamePrefix, "") + csictx.Setenv(ctx, identifiers.EnvKubeNodeName, "") + csictx.Setenv(ctx, identifiers.EnvNodeChrootPath, "") + csictx.Setenv(ctx, identifiers.EnvTmpDir, "") + csictx.Setenv(ctx, identifiers.EnvFCPortsFilterFilePath, "") + csictx.Setenv(ctx, identifiers.EnvEnableCHAP, "") + csictx.Setenv(ctx, identifiers.EnvMaxVolumesPerNode, "42") // ✅ valid value + csictx.Setenv(ctx, identifiers.EnvKubeConfigPath, "myConfigPath") + + opts := getNodeOptions() + if opts.MaxVolumesPerNode != 42 { + t.Errorf("expected MaxVolumesPerNode to be 42, got %d", opts.MaxVolumesPerNode) + } + }) + + t.Run("fallback test with invalid maxVolumesPerNode", func(_ *testing.T) { + ctx := context.Background() + csictx.Setenv(ctx, identifiers.EnvMaxVolumesPerNode, "invalid") // ❌ invalid value + + opts := getNodeOptions() + if opts.MaxVolumesPerNode != 0 { + t.Errorf("expected MaxVolumesPerNode to default to 0, got %d", opts.MaxVolumesPerNode) + } }) } @@ -5437,7 +6156,7 @@ func TestHandleNoLabelMatchRegistration(t *testing.T) { }, setupMocks: func() { getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { - log.Infof("InsideGetArray") + log.Info("InsideGetArray") return map[string]*array.PowerStoreArray{ "Array1": { @@ -5494,7 +6213,7 @@ func TestHandleNoLabelMatchRegistration(t *testing.T) { }, setupMocks: func() { getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { - log.Infof("InsideGetArray") + log.Info("InsideGetArray") return map[string]*array.PowerStoreArray{ "Array1": { @@ -5559,7 +6278,7 @@ func TestHandleNoLabelMatchRegistration(t *testing.T) { } registerHostFunc = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ string, _ []string, _ gopowerstore.HostConnectivityEnum) error { - log.Infof("Inside RegisterHost") + log.Info("Inside RegisterHost") return nil } }, @@ -5584,7 +6303,7 @@ func TestHandleNoLabelMatchRegistration(t *testing.T) { }, setupMocks: func() { getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { - log.Infof("InsideGetArray") + log.Info("InsideGetArray") return map[string]*array.PowerStoreArray{ "Array1": { @@ -5649,7 +6368,7 @@ func TestHandleNoLabelMatchRegistration(t *testing.T) { } registerHostFunc = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ string, _ []string, _ gopowerstore.HostConnectivityEnum) error { - log.Infof("Inside RegisterHost") + log.Info("Inside RegisterHost") return nil } }, @@ -5674,7 +6393,7 @@ func TestHandleNoLabelMatchRegistration(t *testing.T) { }, setupMocks: func() { getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { - log.Infof("InsideGetArray") + log.Info("InsideGetArray") return map[string]*array.PowerStoreArray{ "Array1": { @@ -5739,7 +6458,7 @@ func TestHandleNoLabelMatchRegistration(t *testing.T) { } registerHostFunc = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ string, _ []string, _ gopowerstore.HostConnectivityEnum) error { - log.Infof("Inside RegisterHost") + log.Info("Inside RegisterHost") return nil } }, @@ -5764,7 +6483,7 @@ func TestHandleNoLabelMatchRegistration(t *testing.T) { }, setupMocks: func() { getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { - log.Infof("InsideGetArray") + log.Info("InsideGetArray") return map[string]*array.PowerStoreArray{ "Array1": { @@ -5829,7 +6548,7 @@ func TestHandleNoLabelMatchRegistration(t *testing.T) { } registerHostFunc = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ string, _ []string, _ gopowerstore.HostConnectivityEnum) error { - log.Infof("Inside RegisterHost") + log.Info("Inside RegisterHost") return fmt.Errorf("failed to registerHost") } }, @@ -5854,7 +6573,7 @@ func TestHandleNoLabelMatchRegistration(t *testing.T) { }, setupMocks: func() { getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { - log.Infof("InsideGetArray") + log.Info("InsideGetArray") return map[string]*array.PowerStoreArray{ "Array1": { @@ -5883,7 +6602,7 @@ func TestHandleNoLabelMatchRegistration(t *testing.T) { } getAllRemoteSystemsFunc = func(_ *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { - log.Infof("Inside Remote Systems") + log.Info("Inside Remote Systems") return nil, fmt.Errorf("failed to get remoteSystem") } @@ -5896,7 +6615,7 @@ func TestHandleNoLabelMatchRegistration(t *testing.T) { } registerHostFunc = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ string, _ []string, _ gopowerstore.HostConnectivityEnum) error { - log.Infof("Inside RegisterHost") + log.Info("Inside RegisterHost") return fmt.Errorf("failed to registerHost") } }, @@ -5911,7 +6630,7 @@ func TestHandleNoLabelMatchRegistration(t *testing.T) { mockService := new(MockService) tt.setupMocks() - log.Infof("Test") + log.Info("Test") got, err := mockService.handleNoLabelMatchRegistration(context.Background(), tt.arr, tt.initiators, tt.nodeLabels, tt.arrayAddedList) if (err != nil) != tt.wantErr { @@ -5970,7 +6689,7 @@ func TestHandleLabelMatchRegistration(t *testing.T) { }, setupMocks: func() { getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { - log.Infof("InsideGetArray") + log.Info("InsideGetArray") return map[string]*array.PowerStoreArray{ "Array1": { @@ -6027,7 +6746,7 @@ func TestHandleLabelMatchRegistration(t *testing.T) { }, setupMocks: func() { getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { - log.Infof("InsideGetArray") + log.Info("InsideGetArray") return map[string]*array.PowerStoreArray{ "Array1": { @@ -6092,7 +6811,7 @@ func TestHandleLabelMatchRegistration(t *testing.T) { } registerHostFunc = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ string, _ []string, _ gopowerstore.HostConnectivityEnum) error { - log.Infof("Inside RegisterHost") + log.Info("Inside RegisterHost") return nil } }, @@ -6117,7 +6836,7 @@ func TestHandleLabelMatchRegistration(t *testing.T) { }, setupMocks: func() { getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { - log.Infof("InsideGetArray") + log.Info("InsideGetArray") return map[string]*array.PowerStoreArray{ "Array1": { @@ -6182,7 +6901,7 @@ func TestHandleLabelMatchRegistration(t *testing.T) { } registerHostFunc = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ string, _ []string, _ gopowerstore.HostConnectivityEnum) error { - log.Infof("Inside RegisterHost") + log.Info("Inside RegisterHost") return fmt.Errorf("failed to registerHost") } }, @@ -6207,7 +6926,7 @@ func TestHandleLabelMatchRegistration(t *testing.T) { }, setupMocks: func() { getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { - log.Infof("InsideGetArray") + log.Info("InsideGetArray") return map[string]*array.PowerStoreArray{ "Array1": { @@ -6272,7 +6991,7 @@ func TestHandleLabelMatchRegistration(t *testing.T) { } registerHostFunc = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ string, _ []string, _ gopowerstore.HostConnectivityEnum) error { - log.Infof("Inside RegisterHost") + log.Info("Inside RegisterHost") return fmt.Errorf("failed to registerHost") } }, @@ -6287,7 +7006,7 @@ func TestHandleLabelMatchRegistration(t *testing.T) { mockService := new(MockService) tt.setupMocks() - log.Infof("Test") + log.Info("Test") got, err := mockService.handleLabelMatchRegistration(context.Background(), tt.arr, tt.initiators, tt.nodeLabels, tt.arrayAddedList) if (err != nil) != tt.wantErr { @@ -6296,7 +7015,7 @@ func TestHandleLabelMatchRegistration(t *testing.T) { } if got == tt.want { - log.Infof("Test passed") + log.Info("Test passed") } }) } @@ -6304,20 +7023,46 @@ func TestHandleLabelMatchRegistration(t *testing.T) { // Unit test for createHost func TestService_createHost(t *testing.T) { + defaultK8sConfigFunc := k8sutils.InClusterConfigFunc + defaultK8sClientsetFunc := k8sutils.NewForConfigFunc + + beforeEach := func() { + k8sutils.InClusterConfigFunc = func() (*rest.Config, error) { + return &rest.Config{}, nil + } + k8sutils.NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return fake.NewClientset(), nil + } + + // Base initialize k8sclient + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + }, + }, + }...), + } + } + + afterEach := func() { + k8sutils.InClusterConfigFunc = defaultK8sConfigFunc + k8sutils.NewForConfigFunc = defaultK8sClientsetFunc + } + originalGetArrayfn := getArrayfn originalGetIsHostAlreadyRegistered := getIsHostAlreadyRegistered originalGetAllRemoteSystemsFunc := getAllRemoteSystemsFunc originalGetIsRemoteToOtherArray := getIsRemoteToOtherArray originalRegisterHostFunc := registerHostFunc - nodeLabelsRetrieverMock = new(mocks.NodeLabelsRetrieverInterface) - k8sutils.NodeLabelsRetriever = nodeLabelsRetrieverMock - nodeLabelsRetrieverMock.On("GetNodeLabels", mock.Anything, mock.Anything).Return(map[string]string{"topology.kubernetes.io/zone1": "zone1"}, nil) - nodeLabelsRetrieverMock.On("InClusterConfig", mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("NewForConfig", mock.Anything).Return(nil, nil) clientMock = new(gopowerstoremock.Client) clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) clientMock.On("CreateHost", mock.Anything, mock.Anything). Return(gopowerstore.CreateResponse{ID: validHostID}, nil) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) defer func() { getArrayfn = originalGetArrayfn @@ -6375,7 +7120,11 @@ func TestService_createHost(t *testing.T) { { name: "Successful host creation 1", s: &MockService{ - Service: &Service{}, + Service: &Service{ + opts: Opts{ + KubeNodeName: "node1", + }, + }, }, args: args{ ctx: context.TODO(), @@ -6417,7 +7166,9 @@ func TestService_createHost(t *testing.T) { { name: "Successful host creation 2", s: &MockService{ - Service: &Service{}, + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, }, args: args{ ctx: context.TODO(), @@ -6459,7 +7210,9 @@ func TestService_createHost(t *testing.T) { { name: "Successful host creation 3", s: &MockService{ - Service: &Service{}, + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, }, args: args{ ctx: context.TODO(), @@ -6467,7 +7220,7 @@ func TestService_createHost(t *testing.T) { }, setup: func() { getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { - log.Infof("InsideGetArray") + log.Info("InsideGetArray") return map[string]*array.PowerStoreArray{ "Array1": { @@ -6501,18 +7254,29 @@ func TestService_createHost(t *testing.T) { wantErr: false, }, { - name: "Failure host creation - Label don't match", + name: "Host Registration Success - For New HostConnectivity Secret - LocalOnly", s: &MockService{ - Service: &Service{}, + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, }, args: args{ ctx: context.TODO(), initiators: []string{"initiator1", "initiator2"}, }, setup: func() { + // Override the k8s client + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone": "zone1"}, + }, + }, + }...), + } getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { - log.Infof("InsideGetArray") - return map[string]*array.PowerStoreArray{ "Array1": { Endpoint: "https://10.198.0.1/api/rest", @@ -6521,22 +7285,224 @@ func TestService_createHost(t *testing.T) { Password: "Pass", Insecure: true, BlockProtocol: "auto", - MetroTopology: "Uniform", - Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, - IP: "10.198.0.1", - Client: clientMock, - }, - "Array2": { - Endpoint: "https://10.198.0.2/api/rest", - GlobalID: "Array2", - Username: "admin", - Password: "Pass", - Insecure: true, - BlockProtocol: "auto", - MetroTopology: "Uniform", - Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, - IP: "10.198.0.2", - Client: clientMock, + HostConnectivity: &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + }, + IP: "10.198.0.1", + Client: clientMock, + }, + } + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return false + } + }, + want: []string{"Array1"}, + wantErr: false, + }, + { + name: "Host Registration Success - For New HostConnectivity Secret - Metro ColocatedLocal and Remote", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + // Override the k8s client + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone": "zone1"}, + }, + }, + }...), + } + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + }, + }, + IP: "10.198.0.1", + Client: clientMock, + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + }, + }, + IP: "10.198.0.2", + Client: clientMock, + }, + } + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return false + } + }, + want: []string{"Array1", "Array2"}, + wantErr: false, + }, + { + name: "Host Registration Success - For New HostConnectivity Secret - Metro ColocatedBoth", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + // Override the k8s client + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone": "zone1"}, + }, + }, + }...), + } + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedBoth: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + }, + }, + IP: "10.198.0.1", + Client: clientMock, + }, + } + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return false + } + }, + want: []string{"Array1"}, + wantErr: false, + }, + { + name: "Failure host creation - Label don't match", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + log.Info("InsideGetArray") + + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, + IP: "10.198.0.1", + Client: clientMock, + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, + IP: "10.198.0.2", + Client: clientMock, }, } } @@ -6544,22 +7510,206 @@ func TestService_createHost(t *testing.T) { want: []string{}, wantErr: true, }, + { + name: "Host Registration Failure - For New HostConnectivity Secret - Label don't match", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + // Override k8sclient + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone": "zoneX"}, + }, + }, + }...), + } + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + HostConnectivity: &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"nomatch"}, + }, + }, + }, + }, + }, + }, + IP: "10.198.0.1", + Client: clientMock, + }, + } + } + }, + want: []string{""}, + wantErr: true, + }, + { + name: "Host Registration Failure - For New HostConnectivity Secret - Label duplicated match", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + // Override k8sclient + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone": "zone1"}, + }, + }, + }...), + } + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + ColocatedBoth: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + }, + }, + IP: "10.198.0.1", + Client: clientMock, + }, + } + } + }, + want: []string{""}, + wantErr: true, + }, + { + name: "Host Registration Failure - For New HostConnectivity Secret - Metrotopology set", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + // Override k8sclient + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone": "zoneX"}, + }, + }, + }...), + } + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + HostConnectivity: &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"nomatch"}, + }, + }, + }, + }, + }, + }, + IP: "10.198.0.1", + Client: clientMock, + }, + } + } + }, + want: []string{""}, + wantErr: true, + }, { name: "Failed to get node labels", s: &MockService{ - Service: &Service{}, + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, }, args: args{ ctx: context.TODO(), initiators: []string{"initiator1", "initiator2"}, }, setup: func() { - nodeLabelsRetrieverMock = new(mocks.NodeLabelsRetrieverInterface) - k8sutils.NodeLabelsRetriever = nodeLabelsRetrieverMock - nodeLabelsRetrieverMock.On("GetNodeLabels", mock.Anything, mock.Anything).Return(nil, fmt.Errorf("failed to get node labels")) - nodeLabelsRetrieverMock.On("InClusterConfig", mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("NewForConfig", mock.Anything).Return(nil, nil) - + // Override the k8s client + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset(), + } getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { return map[string]*array.PowerStoreArray{ "Array1": { @@ -6595,19 +7745,15 @@ func TestService_createHost(t *testing.T) { { name: "Host Registration Failure: Both array more than one labels", s: &MockService{ - Service: &Service{}, + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, }, args: args{ ctx: context.TODO(), initiators: []string{"initiator1", "initiator2"}, }, setup: func() { - nodeLabelsRetrieverMock = new(mocks.NodeLabelsRetrieverInterface) - k8sutils.NodeLabelsRetriever = nodeLabelsRetrieverMock - nodeLabelsRetrieverMock.On("GetNodeLabels", mock.Anything, mock.Anything).Return(map[string]string{"topology.kubernetes.io/zone1": "zone1"}, nil) - nodeLabelsRetrieverMock.On("InClusterConfig", mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("NewForConfig", mock.Anything).Return(nil, nil) - getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { return map[string]*array.PowerStoreArray{ "Array1": { @@ -6643,13 +7789,26 @@ func TestService_createHost(t *testing.T) { { name: "Host Registration Failure: One array has more than one labels", s: &MockService{ - Service: &Service{}, + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, }, args: args{ ctx: context.TODO(), initiators: []string{"initiator1", "initiator2"}, }, setup: func() { + // Override the k8s client + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, + }, + }, + }...), + } getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { return map[string]*array.PowerStoreArray{ "Array1": { @@ -6685,13 +7844,26 @@ func TestService_createHost(t *testing.T) { { name: "Failed: To get remote systems when both Array Label matches with Node Labels", s: &MockService{ - Service: &Service{}, + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, }, args: args{ ctx: context.TODO(), initiators: []string{"initiator1", "initiator2"}, }, setup: func() { + // Override the k8s client + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1", "topology.kubernetes.io/zone2": "zone2"}, + }, + }, + }...), + } getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { return map[string]*array.PowerStoreArray{ "Array1": { @@ -6731,7 +7903,9 @@ func TestService_createHost(t *testing.T) { { name: "Failed: To get remote systems when one Array Label matches with Node Labels", s: &MockService{ - Service: &Service{}, + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, }, args: args{ ctx: context.TODO(), @@ -6777,19 +7951,26 @@ func TestService_createHost(t *testing.T) { { name: "Host Registration Failure - Array belongs to different zones", s: &MockService{ - Service: &Service{}, + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, }, args: args{ ctx: context.TODO(), initiators: []string{"initiator1", "initiator2"}, }, setup: func() { - nodeLabelsRetrieverMock = new(mocks.NodeLabelsRetrieverInterface) - k8sutils.NodeLabelsRetriever = nodeLabelsRetrieverMock - nodeLabelsRetrieverMock.On("GetNodeLabels", mock.Anything, mock.Anything).Return(map[string]string{"topology.kubernetes.io/zone1": "zone1", "topology.kubernetes.io/zone2": "zone2"}, nil) - nodeLabelsRetrieverMock.On("InClusterConfig", mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("NewForConfig", mock.Anything).Return(nil, nil) - + // Override the k8s client + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1", "topology.kubernetes.io/zone2": "zone2"}, + }, + }, + }...), + } getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { return map[string]*array.PowerStoreArray{ "Array1": { @@ -6852,19 +8033,26 @@ func TestService_createHost(t *testing.T) { { name: "Successful Host Registration with Co-Local and Co-remote", s: &MockService{ - Service: &Service{}, + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, }, args: args{ ctx: context.TODO(), initiators: []string{"initiator1", "initiator2"}, }, setup: func() { - nodeLabelsRetrieverMock = new(mocks.NodeLabelsRetrieverInterface) - k8sutils.NodeLabelsRetriever = nodeLabelsRetrieverMock - nodeLabelsRetrieverMock.On("GetNodeLabels", mock.Anything, mock.Anything).Return(map[string]string{"topology.kubernetes.io/zone2": "zone2"}, nil) - nodeLabelsRetrieverMock.On("InClusterConfig", mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("NewForConfig", mock.Anything).Return(nil, nil) - + // Override the k8s client + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone2": "zone2"}, + }, + }, + }...), + } getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { return map[string]*array.PowerStoreArray{ "Array1": { @@ -6927,19 +8115,15 @@ func TestService_createHost(t *testing.T) { { name: "Host Registration Failure - Host Already registerd", s: &MockService{ - Service: &Service{}, + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, }, args: args{ ctx: context.TODO(), initiators: []string{"initiator1", "initiator2"}, }, setup: func() { - nodeLabelsRetrieverMock = new(mocks.NodeLabelsRetrieverInterface) - k8sutils.NodeLabelsRetriever = nodeLabelsRetrieverMock - nodeLabelsRetrieverMock.On("GetNodeLabels", mock.Anything, mock.Anything).Return(map[string]string{"topology.kubernetes.io/zone1": "zone1"}, nil) - nodeLabelsRetrieverMock.On("InClusterConfig", mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("NewForConfig", mock.Anything).Return(nil, nil) - getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { return map[string]*array.PowerStoreArray{ "Array1": { @@ -6980,10 +8164,61 @@ func TestService_createHost(t *testing.T) { want: []string{"Array1", "Array2"}, wantErr: false, }, + { + name: "Host Registration Failure - Host Already registerd with hostconnectivity", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + HostConnectivity: &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + }, + IP: "10.198.0.1", + Client: clientMock, + }, + } + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return true + } + }, + want: []string{"Array1"}, + wantErr: false, + }, { name: "Host Registration Failure - Create Host API fail", s: &MockService{ - Service: &Service{}, + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, }, args: args{ ctx: context.TODO(), @@ -7054,6 +8289,8 @@ func TestService_createHost(t *testing.T) { clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) clientMock.On("CreateHost", mock.Anything, mock.Anything). Return(gopowerstore.CreateResponse{}, fmt.Errorf("failed to create host")) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) }, want: []string{}, wantErr: true, @@ -7061,7 +8298,9 @@ func TestService_createHost(t *testing.T) { { name: "Host Registration Failure with Local only", s: &MockService{ - Service: &Service{}, + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, }, args: args{ ctx: context.TODO(), @@ -7090,6 +8329,8 @@ func TestService_createHost(t *testing.T) { clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) clientMock.On("CreateHost", mock.Anything, mock.Anything). Return(gopowerstore.CreateResponse{}, fmt.Errorf("failed to create host")) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) }, want: []string{}, wantErr: true, @@ -7097,7 +8338,9 @@ func TestService_createHost(t *testing.T) { { name: "Host Registration Failure - getIsRemoteToOtherArray", s: &MockService{ - Service: &Service{}, + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, }, args: args{ ctx: context.TODO(), @@ -7105,7 +8348,7 @@ func TestService_createHost(t *testing.T) { }, setup: func() { getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { - log.Infof("InsideGetArray") + log.Info("InsideGetArray") return map[string]*array.PowerStoreArray{ "Array1": { @@ -7171,6 +8414,8 @@ func TestService_createHost(t *testing.T) { clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) clientMock.On("CreateHost", mock.Anything, mock.Anything). Return(gopowerstore.CreateResponse{ID: validHostID}, nil) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) getIsRemoteToOtherArray = func(_ *Service, _ context.Context, _, _ *array.PowerStoreArray) bool { return false @@ -7182,7 +8427,9 @@ func TestService_createHost(t *testing.T) { { name: "Host Registration Failure - Register Host fail", s: &MockService{ - Service: &Service{}, + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, }, args: args{ ctx: context.TODO(), @@ -7253,6 +8500,8 @@ func TestService_createHost(t *testing.T) { clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) clientMock.On("CreateHost", mock.Anything, mock.Anything). Return(gopowerstore.CreateResponse{}, fmt.Errorf("failed to create host")) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) registerHostFunc = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ string, _ []string, _ gopowerstore.HostConnectivityEnum) error { return fmt.Errorf("failed to register Host") @@ -7265,6 +8514,8 @@ func TestService_createHost(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + beforeEach() + defer afterEach() tt.setup() got, err := tt.s.createHost(tt.args.ctx, tt.args.initiators) @@ -7385,3 +8636,390 @@ func elementsMatch(a, b []string) bool { } return true } + +func TestExtractPort(t *testing.T) { + tests := []struct { + name string + url string + expected string + expectError bool + }{ + { + name: "Valid URL with port", + url: "http://localhost:8080", + expected: "8080", + expectError: false, + }, + { + name: "Valid URL without port", + url: "http://localhost", + expected: "", + expectError: true, + }, + { + name: "Invalid URL format", + url: "://bad-url", + expected: "", + expectError: true, + }, + { + name: "HTTPS URL with port", + url: "https://example.com:443", + expected: "443", + expectError: false, + }, + { + name: "URL with path and port", + url: "http://example.com:9000/path", + expected: "9000", + expectError: false, + }, + { + name: "Empty string input", + url: "", + expected: "", + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + port, err := ExtractPort(tt.url) + + if tt.expectError { + if err == nil { + t.Errorf("Expected error, got none. Port: %s", port) + } + } else { + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if port != tt.expected { + t.Errorf("Expected port: %s, got: %s", tt.expected, port) + } + } + }) + } +} + +func TestService_updateHost(t *testing.T) { + tests := []struct { + name string // description of this test case + // Named input parameters for target function. + initiators []string + client gopowerstore.Client + host gopowerstore.Host + arrayID string + connectivity *gopowerstore.HostConnectivityEnum + wantErr bool + }{ + { + name: "Update host", + initiators: []string{}, + client: nil, + host: gopowerstore.Host{}, + arrayID: "f16c:f7ec:cfa2:e1c5:9a3c:cb08:801f:36b8", + connectivity: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // TODO: construct the receiver type. + var s Service + gotErr := s.updateHost(context.Background(), tt.initiators, tt.client, tt.host, tt.arrayID, tt.connectivity) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("updateHost() failed: %v", gotErr) + } + return + } + if tt.wantErr { + t.Fatal("updateHost() succeeded unexpectedly") + } + }) + } +} + +func TestMetroMatchNodeSelectorTerms(t *testing.T) { + tests := []struct { + name string + terms []k8score.NodeSelectorTerm + nodeLabels map[string]string + wantMatch bool + wantLabels map[string]string + }{ + { + name: "Match with NodeSelectorOpIn", + terms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"us-east-1a", "us-east-1b"}, + }, + }, + }, + }, + nodeLabels: map[string]string{"zone": "us-east-1a"}, + wantMatch: true, + wantLabels: map[string]string{"zone": "us-east-1a"}, + }, + { + name: "Mismatch with NodeSelectorOpIn", + terms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"us-west-1a"}, + }, + }, + }, + }, + nodeLabels: map[string]string{"zone": "us-east-1a"}, + wantMatch: false, + wantLabels: nil, + }, + { + name: "Match with NodeSelectorOpExists", + terms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "diskType", + Operator: k8score.NodeSelectorOpExists, + }, + }, + }, + }, + nodeLabels: map[string]string{"diskType": "ssd"}, + wantMatch: true, + wantLabels: map[string]string{"diskType": "ssd"}, + }, + { + name: "Mismatch with NodeSelectorOpDoesNotExist", + terms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "gpu", + Operator: k8score.NodeSelectorOpDoesNotExist, + }, + }, + }, + }, + nodeLabels: map[string]string{"gpu": "nvidia"}, + wantMatch: false, + wantLabels: nil, + }, + { + name: "Match with NodeSelectorOpNotIn", + terms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "env", + Operator: k8score.NodeSelectorOpNotIn, + Values: []string{"prod"}, + }, + }, + }, + }, + nodeLabels: map[string]string{"env": "dev"}, + wantMatch: true, + wantLabels: map[string]string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotMatch, gotLabels := metroMatchNodeSelectorTerms(tt.terms, tt.nodeLabels) + assert.Equal(t, tt.wantMatch, gotMatch) + assert.Equal(t, tt.wantLabels, gotLabels) + }) + } +} + +func TestService_setupHost(t *testing.T) { + tests := []struct { + name string + // Named input parameters for target function. + initiators []string + client gopowerstore.Client + arrayIP string + arrayID string + wantErr bool + }{ + { + name: "Setup host", + initiators: []string{}, + client: nil, + arrayIP: "f16c:f7ec:cfa2:e1c5:9a3c:cb08:801f:36b8", + arrayID: "f16c:f7ec:cfa2:e1c5:9a3c:cb08:801f:36b8", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var s Service + gotErr := s.setupHost(tt.initiators, tt.client, tt.arrayIP, tt.arrayID) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("setupHost() failed: %v", gotErr) + } + return + } + if tt.wantErr { + t.Fatal("setupHost() succeeded unexpectedly") + } + }) + } +} + +func TestIsHostAlreadyRegistered(t *testing.T) { + tests := []struct { + name string + // Named input parameters for target function. + initiators []string + client gopowerstore.Client + before func(*gopowerstoremock.Client) + wantResult bool + }{ + { + name: "IsHostAlreadyRegistered - true", + initiators: []string{"my-port"}, + client: new(gopowerstoremock.Client), + before: func(client *gopowerstoremock.Client) { + client.On("GetHosts", mock.Anything).Return( + []gopowerstore.Host{{ + ID: "host-id", + Initiators: []gopowerstore.InitiatorInstance{{ + PortName: "my-port", + PortType: gopowerstore.InitiatorProtocolTypeEnumISCSI, + }}, + Name: "host-name", + }}, nil) + }, + wantResult: true, + }, + { + name: "IsHostAlreadyRegistered - not found", + initiators: []string{}, + client: new(gopowerstoremock.Client), + before: func(client *gopowerstoremock.Client) { + client.On("GetHosts", mock.Anything).Return( + []gopowerstore.Host{{ + ID: "host-id", + Initiators: []gopowerstore.InitiatorInstance{{ + PortName: "my-port", + PortType: gopowerstore.InitiatorProtocolTypeEnumISCSI, + }}, + Name: "host-name", + }}, nil) + }, + wantResult: false, + }, + { + name: "IsHostAlreadyRegistered - unable to get hosts", + initiators: []string{}, + client: new(gopowerstoremock.Client), + before: func(client *gopowerstoremock.Client) { + client.On("GetHosts", mock.Anything).Return( + []gopowerstore.Host{}, fmt.Errorf("unable to get hosts")) + }, + wantResult: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var s Service + if tt.before != nil { + tt.before(tt.client.(*gopowerstoremock.Client)) + } + gotResult := s.isHostAlreadyRegistered(context.Background(), tt.client, tt.initiators) + if gotResult != tt.wantResult { + t.Errorf("isHostAlreadyRegistered() = %v, want %v", gotResult, tt.wantResult) + } + }) + } +} + +func TestRemoveRemnantMounts(t *testing.T) { + t.Run("fails to get remnant target mounts", func(t *testing.T) { + mockFs := new(mocks.FsInterface) + mockFs.On("ReadFile", mock.Anything).Return(nil, fmt.Errorf("error")) + + _, err := removeRemnantMounts(context.Background(), "/var/lib/test", mockFs, csmlog.Fields{}) + if err == nil { + t.Errorf("expected an error, got nil") + } + }) + + t.Run("no remnant target mounts", func(t *testing.T) { + mockFs := new(mocks.FsInterface) + mockFs.On("ReadFile", mock.Anything).Return([]byte("data"), nil) + mockFs.On("ParseProcMounts", mock.Anything, mock.Anything).Return([]gofsutil.Info{ + { + Path: "/var/lib/other", + }, + }, nil) + _, err := removeRemnantMounts(context.Background(), "/var/lib/test", mockFs, csmlog.Fields{}) + if err == nil { + t.Errorf("expected an error, got nil") + } + }) +} + +func TestCountActiveSessionsInitiators(t *testing.T) { + tests := []struct { + name string + host gopowerstore.Host + wantCount int + }{ + { + name: "single initiator with one ActiveSessions", + host: gopowerstore.Host{ + Initiators: []gopowerstore.InitiatorInstance{ + {ActiveSessions: []gopowerstore.ActiveSessionInstance{{ApplianceID: "s1"}}}, + {ActiveSessions: nil}, + {ActiveSessions: []gopowerstore.ActiveSessionInstance{}}, + }, + }, + wantCount: 1, + }, + { + name: "single initiator with two ActiveSessions", + host: gopowerstore.Host{ + Initiators: []gopowerstore.InitiatorInstance{ + {ActiveSessions: []gopowerstore.ActiveSessionInstance{{ApplianceID: "s1"}}}, + {ActiveSessions: nil}, + {ActiveSessions: []gopowerstore.ActiveSessionInstance{{ApplianceID: "s2"}}}, + }, + }, + wantCount: 2, + }, + { + name: "single initiator with no ActiveSessions", + host: gopowerstore.Host{ + Initiators: []gopowerstore.InitiatorInstance{ + {ActiveSessions: nil}, + {ActiveSessions: nil}, + {ActiveSessions: []gopowerstore.ActiveSessionInstance{}}, + }, + }, + wantCount: 0, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + numOfInitiatorsWithActiveSession := countActiveSessionsInitiators(tc.host) + if numOfInitiatorsWithActiveSession != tc.wantCount { + t.Errorf("countActiveSessionsInitiators() = %d, want %d", numOfInitiatorsWithActiveSession, tc.wantCount) + } + }) + } +} diff --git a/pkg/node/publisher.go b/pkg/node/publisher.go index 11368597..966cbebe 100644 --- a/pkg/node/publisher.go +++ b/pkg/node/publisher.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,17 +21,17 @@ package node import ( "context" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/pkg/identifiers" "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" - log "github.com/sirupsen/logrus" + "github.com/dell/csmlog" + "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) // VolumePublisher allows to node publish a volume type VolumePublisher interface { - Publish(ctx context.Context, logFields log.Fields, fs fs.Interface, + Publish(ctx context.Context, logFields csmlog.Fields, fs fs.Interface, vc *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) } @@ -41,7 +41,7 @@ type SCSIPublisher struct { } // Publish publishes volume as either raw block or mount by mounting it to the target path -func (sp *SCSIPublisher) Publish(ctx context.Context, logFields log.Fields, fs fs.Interface, vc *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) { +func (sp *SCSIPublisher) Publish(ctx context.Context, logFields csmlog.Fields, fs fs.Interface, vc *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) { published, err := isAlreadyPublished(ctx, targetPath, getRWModeString(isRO), fs) if err != nil { return nil, err @@ -57,8 +57,9 @@ func (sp *SCSIPublisher) Publish(ctx context.Context, logFields log.Fields, fs f return sp.publishMount(ctx, logFields, fs, vc, isRO, targetPath, stagingPath) } -func (sp *SCSIPublisher) publishBlock(ctx context.Context, logFields log.Fields, fs fs.Interface, _ *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) { - log.WithFields(logFields).Info("start publishing as block device") +func (sp *SCSIPublisher) publishBlock(ctx context.Context, logFields csmlog.Fields, fs fs.Interface, _ *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) { + log := log.WithFields(logFields).WithContext(ctx) + log.Info("start publishing as block device") if isRO { return nil, status.Error(codes.InvalidArgument, "read only not supported for Block Volume") @@ -68,26 +69,26 @@ func (sp *SCSIPublisher) publishBlock(ctx context.Context, logFields log.Fields, return nil, status.Errorf(codes.Internal, "can't create target file %s: %s", targetPath, err.Error()) } - log.WithFields(logFields).Info("target path successfully created") + log.Info("target path successfully created") if err := fs.GetUtil().BindMount(ctx, stagingPath, targetPath); err != nil { return nil, status.Errorf(codes.Internal, "error bind disk %s to target path: %s", stagingPath, err.Error()) } - log.WithFields(logFields).Info("volume successfully binded") + log.Info("volume successfully binded") return &csi.NodePublishVolumeResponse{}, nil } -func (sp *SCSIPublisher) publishMount(ctx context.Context, logFields log.Fields, fs fs.Interface, vc *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) { +func (sp *SCSIPublisher) publishMount(ctx context.Context, logFields csmlog.Fields, fs fs.Interface, vc *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) { + log := log.WithFields(logFields).WithContext(ctx) if vc.GetAccessMode().GetMode() == csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER { - // MULTI_WRITER not supported for mount volumes - return nil, status.Error(codes.Unimplemented, "Mount volumes do not support AccessMode MULTI_NODE_MULTI_WRITER") + log.Infof(" Mount volume with the AccessMode ReadWriteMany") } if vc.GetAccessMode().GetMode() == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY { // Warning in case of MULTI_NODE_READER_ONLY for mount volumes - log.Warningf("Mount volume with the AccessMode ReadOnlyMany") + log.Warnf("Mount volume with the AccessMode ReadOnlyMany") } var opts []string @@ -106,7 +107,7 @@ func (sp *SCSIPublisher) publishMount(ctx context.Context, logFields log.Fields, "can't create target dir with Mkdirall %s: %s", targetPath, err.Error()) } - log.WithFields(logFields).Info("target dir successfully created") + log.Info("target dir successfully created") curFS, err := fs.GetUtil().GetDiskFormat(ctx, stagingPath) if err != nil { @@ -121,7 +122,7 @@ func (sp *SCSIPublisher) publishMount(ctx context.Context, logFields log.Fields, } if curFS == "" { - log.WithFields(logFields).Infof("no filesystem found on staged disk %s", stagingPath) + log.Infof("no filesystem found on staged disk : %s", stagingPath) if isRO { return nil, status.Errorf(codes.FailedPrecondition, "RO mount required but no fs detected on staged volume %s", stagingPath) @@ -131,7 +132,7 @@ func (sp *SCSIPublisher) publishMount(ctx context.Context, logFields log.Fields, return nil, status.Errorf(codes.Internal, "can't format staged device %s: %s", stagingPath, err.Error()) } - log.WithFields(logFields).Infof("staged disk %s successfully formatted to %s", stagingPath, targetFS) + log.Infof("staged disk %s successfully formatted to %s", stagingPath, targetFS) } if isRO { mntFlags = append(mntFlags, "ro") @@ -141,7 +142,7 @@ func (sp *SCSIPublisher) publishMount(ctx context.Context, logFields log.Fields, return nil, status.Errorf(codes.Internal, "error performing mount for staging path %s: %s", stagingPath, err.Error()) } - log.WithFields(logFields).Info("volume successfully mounted") + log.Info("volume successfully mounted") return &csi.NodePublishVolumeResponse{}, nil } @@ -150,9 +151,10 @@ func (sp *SCSIPublisher) publishMount(ctx context.Context, logFields log.Fields, type NFSPublisher struct{} // Publish publishes nfs volume by mounting it to the target path -func (np *NFSPublisher) Publish(ctx context.Context, logFields log.Fields, fs fs.Interface, +func (np *NFSPublisher) Publish(ctx context.Context, logFields csmlog.Fields, fs fs.Interface, vc *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string, ) (*csi.NodePublishVolumeResponse, error) { + log := log.WithFields(logFields).WithContext(ctx) published, err := isAlreadyPublished(ctx, targetPath, getRWModeString(isRO), fs) if err != nil { return nil, err @@ -166,7 +168,7 @@ func (np *NFSPublisher) Publish(ctx context.Context, logFields log.Fields, fs fs return nil, status.Errorf(codes.Internal, "can't create target folder %s: %s", stagingPath, err.Error()) } - log.WithFields(logFields).Info("target path successfully created") + log.Info("target path successfully created") mntFlags := identifiers.GetMountFlags(vc) @@ -179,6 +181,6 @@ func (np *NFSPublisher) Publish(ctx context.Context, logFields log.Fields, fs fs "error bind disk %s to target path: %s", stagingPath, err.Error()) } - log.WithFields(logFields).Info("volume successfully binded") + log.Info("volume successfully binded") return &csi.NodePublishVolumeResponse{}, nil } diff --git a/pkg/node/stager.go b/pkg/node/stager.go index 8a896784..01d824fa 100644 --- a/pkg/node/stager.go +++ b/pkg/node/stager.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2024 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,13 +29,13 @@ import ( "strings" "time" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/pkg/array" "github.com/dell/csi-powerstore/v2/pkg/identifiers" "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + "github.com/dell/csmlog" "github.com/dell/gobrick" "github.com/dell/gopowerstore" - log "github.com/sirupsen/logrus" + "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -47,7 +47,7 @@ const ( // VolumeStager allows to node stage a volume type VolumeStager interface { - Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, logFields log.Fields, fs fs.Interface, id string, isRemote bool, client gopowerstore.Client) (*csi.NodeStageVolumeResponse, error) + Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, stagingPath string, nodeID string, logFields csmlog.Fields, fs fs.Interface, id string, isRemote bool, client gopowerstore.Client) (*csi.NodeStageVolumeResponse, error) } // ReachableEndPoint checks if the endpoint is reachable or not @@ -63,13 +63,15 @@ type SCSIStager struct { } // Stage stages volume by connecting it through either FC or iSCSI and creating bind mount to staging path -func (s *SCSIStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, - logFields log.Fields, fs fs.Interface, id string, isRemote bool, client gopowerstore.Client, +func (s *SCSIStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, stagingPath string, nodeID string, + logFields csmlog.Fields, fs fs.Interface, id string, isRemote bool, client gopowerstore.Client, ) (*csi.NodeStageVolumeResponse, error) { - stagingPath := req.GetStagingTargetPath() + log := log.WithContext(ctx) orginalContext := req.PublishContext - id, stagingPath = getStagingPath(ctx, stagingPath, id) volume, err := client.GetVolume(ctx, id) + if err != nil { + return nil, err + } targetMap := make(map[string]string) err = s.AddTargetsInfoToMap(targetMap, volume.ApplianceID, client, isRemote) if err != nil { @@ -77,11 +79,29 @@ func (s *SCSIStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, } if !isRemote { - targetMap[identifiers.TargetMapDeviceWWN] = orginalContext[identifiers.TargetMapDeviceWWN] - targetMap[identifiers.TargetMapLUNAddress] = orginalContext[identifiers.TargetMapLUNAddress] + wwn, ok := orginalContext[identifiers.TargetMapDeviceWWN] + lun, ok := orginalContext[identifiers.TargetMapLUNAddress] + if !ok { + wwn = strings.TrimPrefix(volume.Wwn, identifiers.WWNPrefix) + lun, err = getLunAddressFromArray(ctx, client, id, nodeID) + if err != nil { + return nil, err + } + } + targetMap[identifiers.TargetMapDeviceWWN] = wwn + targetMap[identifiers.TargetMapLUNAddress] = lun } else { - targetMap[identifiers.TargetMapRemoteDeviceWWN] = orginalContext[identifiers.TargetMapRemoteDeviceWWN] - targetMap[identifiers.TargetMapRemoteLUNAddress] = orginalContext[identifiers.TargetMapRemoteLUNAddress] + wwn, ok := orginalContext[identifiers.TargetMapRemoteDeviceWWN] + lun, ok := orginalContext[identifiers.TargetMapRemoteLUNAddress] + if !ok { + wwn = strings.TrimPrefix(volume.Wwn, identifiers.WWNPrefix) + lun, err = getLunAddressFromArray(ctx, client, id, nodeID) + if err != nil { + return nil, err + } + } + targetMap[identifiers.TargetMapRemoteDeviceWWN] = wwn + targetMap[identifiers.TargetMapRemoteLUNAddress] = lun } publishContext, err := readSCSIInfoFromPublishContext(targetMap, s.useFC, s.useNVME, isRemote) @@ -102,7 +122,7 @@ func (s *SCSIStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, logFields["WWN"] = publishContext.deviceWWN logFields["Lun"] = publishContext.volumeLUNAddress logFields["StagingPath"] = stagingPath - ctx = identifiers.SetLogFields(ctx, logFields) + ctx = csmlog.SetLogFields(ctx, logFields) found, ready, err := isReadyToPublish(ctx, stagingPath, fs) if err != nil { @@ -112,8 +132,7 @@ func (s *SCSIStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, log.WithFields(logFields).Info("device already staged") return &csi.NodeStageVolumeResponse{}, nil } else if found { - log.WithFields(logFields).Warning("volume found in staging path but it is not ready for publish," + - "try to unmount it and retry staging again") + log.WithFields(logFields).Warn("volume found in staging path but it is not ready for publish, try to unmount it and retry staging again") _, err := unstageVolume(ctx, stagingPath, id, logFields, err, fs) if err != nil { return nil, status.Errorf(codes.Internal, "failed to unmount volume: %s", err.Error()) @@ -144,19 +163,39 @@ func (s *SCSIStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, return &csi.NodeStageVolumeResponse{}, nil } +func getLunAddressFromArray(ctx context.Context, client gopowerstore.Client, id string, nodeID string) (string, error) { + log := log.WithContext(ctx) + log.Infof("GetHostVolumeMappingByVolumeID for volId %s host %s", id, nodeID) + var node gopowerstore.Host + node, err := client.GetHostByName(ctx, nodeID) + if err != nil { + return "", status.Errorf(codes.Internal, + "failed to find host '%s' during staging: %s", nodeID, err.Error()) + } + mapping, err := client.GetHostVolumeMappingByVolumeID(ctx, id) + if err != nil { + return "", status.Errorf(codes.Internal, + "failed to get mapping for volume with ID '%s' during staging: %s", id, err.Error()) + } + for _, m := range mapping { + if m.HostID == node.ID { + lun := strconv.FormatInt(m.LogicalUnitNumber, 10) + return lun, nil + } + } + return "", status.Errorf(codes.Internal, + "failed to get LUN for volume with ID '%s' during staging", id) +} + // NFSStager implementation of NodeVolumeStager for NFS volumes type NFSStager struct { array *array.PowerStoreArray } // Stage stages volume by mounting volumes as nfs to the staging path -func (n *NFSStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, - logFields log.Fields, fs fs.Interface, id string, _ bool, _ gopowerstore.Client, +func (n *NFSStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, stagingPath string, _ string, + logFields csmlog.Fields, fs fs.Interface, id string, _ bool, _ gopowerstore.Client, ) (*csi.NodeStageVolumeResponse, error) { - stagingPath := req.GetStagingTargetPath() - - id, stagingPath = getStagingPath(ctx, stagingPath, id) - hostIP := req.PublishContext[identifiers.KeyHostIP] exportID := req.PublishContext[identifiers.KeyExportID] nfsExport := req.PublishContext[identifiers.KeyNfsExportPath] @@ -177,7 +216,7 @@ func (n *NFSStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, logFields["NatIP"] = natIP logFields["NFSv4ACLs"] = req.PublishContext[identifiers.KeyNfsACL] logFields["NasName"] = nasName - ctx = identifiers.SetLogFields(ctx, logFields) + log := log.WithContext(ctx).WithFields(logFields) found, err := isReadyToPublishNFS(ctx, stagingPath, fs) if err != nil { @@ -185,7 +224,7 @@ func (n *NFSStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, } if found { - log.WithFields(logFields).Info("device already staged") + log.Info("device already staged") return &csi.NodeStageVolumeResponse{}, nil } @@ -193,7 +232,7 @@ func (n *NFSStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, return nil, status.Errorf(codes.Internal, "can't create target folder %s: %s", stagingPath, err.Error()) } - log.WithFields(logFields).Info("stage path successfully created") + log.Info("stage path successfully created") mntFlags := identifiers.GetMountFlags(req.GetVolumeCapability()) if err := fs.GetUtil().Mount(ctx, nfsExport, stagingPath, "", mntFlags...); err != nil { @@ -216,7 +255,7 @@ func (n *NFSStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, if err == nil { mode = os.FileMode(perm) // #nosec: G115 false positive } else { - log.WithFields(logFields).Warn("can't parse file mode, invalid mode specified. Default mode permissions will be set.") + log.Warn("can't parse file mode, invalid mode specified. Default mode permissions will be set.") } } else { aclsConfigured, err = validateAndSetACLs(ctx, &NFSv4ACLs{}, nasName, n.array.GetClient(), acls, filepath.Join(stagingPath, commonNfsVolumeFolder)) @@ -234,12 +273,14 @@ func (n *NFSStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, } if allowRoot == "false" { - log.WithFields(logFields).Info("removing allow root from nfs export") + log.Info("removing allow root from nfs export") var hostsToRemove []string var hostsToAdd []string - hostsToRemove = append(hostsToRemove, hostIP+"/255.255.255.255") - hostsToAdd = append(hostsToAdd, hostIP) + if hostIP != "" { + hostsToRemove = append(hostsToRemove, hostIP+"/255.255.255.255") + hostsToAdd = append(hostsToAdd, hostIP) + } if natIP != "" { hostsToRemove = append(hostsToRemove, natIP) @@ -258,7 +299,7 @@ func (n *NFSStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, } } - log.WithFields(logFields).Info("nfs share successfully mounted") + log.Info("nfs share successfully mounted") return &csi.NodeStageVolumeResponse{}, nil } @@ -415,11 +456,11 @@ func readFCTargetsFromPublishContext(pc map[string]string, isRemote bool) []gobr } func (s *SCSIStager) connectDevice(ctx context.Context, data scsiPublishContextData) (string, error) { - logFields := identifiers.GetLogFields(ctx) + log := log.WithContext(ctx) var err error lun, err := strconv.Atoi(data.volumeLUNAddress) if err != nil { - log.WithFields(logFields).Errorf("failed to convert lun number to int: %s", err.Error()) + log.Errorf("failed to convert lun number to int: %s", err.Error()) return "", status.Errorf(codes.Internal, "failed to convert lun number to int: %s", err.Error()) } @@ -434,7 +475,7 @@ func (s *SCSIStager) connectDevice(ctx context.Context, data scsiPublishContextD } if err != nil { - log.WithFields(logFields).Errorf("Unable to find device after multiple discovery attempts: %s", err.Error()) + log.Errorf("Unable to find device after multiple discovery attempts: %s", err.Error()) return "", status.Errorf(codes.Internal, "unable to find device after multiple discovery attempts: %s", err.Error()) } @@ -445,7 +486,7 @@ func (s *SCSIStager) connectDevice(ctx context.Context, data scsiPublishContextD func (s *SCSIStager) connectISCSIDevice(ctx context.Context, lun int, data scsiPublishContextData, ) (gobrick.Device, error) { - logFields := identifiers.GetLogFields(ctx) + logFields := csmlog.ExtractFieldsFromContext(ctx) var targets []gobrick.ISCSITargetInfo for _, t := range data.iscsiTargets { targets = append(targets, gobrick.ISCSITargetInfo{Target: t.Target, Portal: t.Portal}) @@ -454,7 +495,7 @@ func (s *SCSIStager) connectISCSIDevice(ctx context.Context, connectorCtx, cFunc := context.WithTimeout(context.Background(), time.Second*120) defer cFunc() - connectorCtx = identifiers.SetLogFields(connectorCtx, logFields) + connectorCtx = csmlog.SetLogFields(connectorCtx, logFields) return s.iscsiConnector.ConnectVolume(connectorCtx, gobrick.ISCSIVolumeInfo{ Targets: targets, Lun: lun, @@ -464,7 +505,7 @@ func (s *SCSIStager) connectISCSIDevice(ctx context.Context, func (s *SCSIStager) connectNVMEDevice(ctx context.Context, wwn string, data scsiPublishContextData, useFC bool, ) (gobrick.Device, error) { - logFields := identifiers.GetLogFields(ctx) + logFields := csmlog.ExtractFieldsFromContext(ctx) var targets []gobrick.NVMeTargetInfo if useFC { @@ -480,7 +521,7 @@ func (s *SCSIStager) connectNVMEDevice(ctx context.Context, connectorCtx, cFunc := context.WithTimeout(context.Background(), time.Second*120) defer cFunc() - connectorCtx = identifiers.SetLogFields(connectorCtx, logFields) + connectorCtx = csmlog.SetLogFields(connectorCtx, logFields) return s.nvmeConnector.ConnectVolume(connectorCtx, gobrick.NVMeVolumeInfo{ Targets: targets, WWN: wwn, @@ -490,7 +531,7 @@ func (s *SCSIStager) connectNVMEDevice(ctx context.Context, func (s *SCSIStager) connectFCDevice(ctx context.Context, lun int, data scsiPublishContextData, ) (gobrick.Device, error) { - logFields := identifiers.GetLogFields(ctx) + logFields := csmlog.ExtractFieldsFromContext(ctx) var targets []gobrick.FCTargetInfo for _, t := range data.fcTargets { @@ -500,7 +541,7 @@ func (s *SCSIStager) connectFCDevice(ctx context.Context, connectorCtx, cFunc := context.WithTimeout(context.Background(), time.Second*120) defer cFunc() - connectorCtx = identifiers.SetLogFields(connectorCtx, logFields) + connectorCtx = csmlog.SetLogFields(connectorCtx, logFields) return s.fcConnector.ConnectVolume(connectorCtx, gobrick.FCVolumeInfo{ Targets: targets, Lun: lun, @@ -508,18 +549,18 @@ func (s *SCSIStager) connectFCDevice(ctx context.Context, } func isReadyToPublish(ctx context.Context, stagingPath string, fs fs.Interface) (bool, bool, error) { - logFields := identifiers.GetLogFields(ctx) + log := log.WithContext(ctx) stageInfo, found, err := getTargetMount(ctx, stagingPath, fs) if err != nil { return found, false, err } if !found { - log.WithFields(logFields).Warning("staged device not found") + log.Warn("staged device not found") return found, false, nil } if strings.HasSuffix(stageInfo.Source, "deleted") { - log.WithFields(logFields).Warning("staged device linked with deleted path") + log.Warn("staged device linked with deleted path") return found, false, nil } @@ -531,18 +572,18 @@ func isReadyToPublish(ctx context.Context, stagingPath string, fs fs.Interface) } func isReadyToPublishNFS(ctx context.Context, stagingPath string, fs fs.Interface) (bool, error) { - logFields := identifiers.GetLogFields(ctx) + log := log.WithContext(ctx) stageInfo, found, err := getTargetMount(ctx, stagingPath, fs) if err != nil { return found, err } if !found { - log.WithFields(logFields).Warning("staged device not found") + log.Warn("staged device not found") return found, nil } if strings.HasSuffix(stageInfo.Source, "deleted") { - log.WithFields(logFields).Warning("staged device linked with deleted path") + log.Warn("staged device linked with deleted path") return found, nil } @@ -571,7 +612,7 @@ func (s *SCSIStager) AddTargetsInfoToMap( iscsiTargetsInfo, err := identifiers.GetISCSITargetsInfoFromStorage(client, volumeApplianceID) if err != nil { - log.Error("error unable to get iSCSI targets from array", err) + log.Errorf("error unable to get iSCSI targets from array %s", err.Error()) } for i, t := range iscsiTargetsInfo { targetMap[fmt.Sprintf("%s%d", iscsiPortalsKey, i)] = t.Portal @@ -579,7 +620,7 @@ func (s *SCSIStager) AddTargetsInfoToMap( } fcTargetsInfo, err := identifiers.GetFCTargetsInfoFromStorage(client, volumeApplianceID) if err != nil { - log.Error("error unable to get FC targets from array", err) + log.Errorf("error unable to get FC targets from array %s", err.Error()) } for i, t := range fcTargetsInfo { targetMap[fmt.Sprintf("%s%d", fcWwpnKey, i)] = t.WWPN @@ -587,7 +628,7 @@ func (s *SCSIStager) AddTargetsInfoToMap( nvmefcTargetInfo, err := identifiers.GetNVMEFCTargetInfoFromStorage(client, volumeApplianceID) if err != nil { - log.Error("error unable to get NVMeFC targets from array", err) + log.Errorf("error unable to get NVMeFC targets from array %s", err.Error()) } for i, t := range nvmefcTargetInfo { targetMap[fmt.Sprintf("%s%d", nvmeFcPortalsKey, i)] = t.Portal @@ -596,7 +637,7 @@ func (s *SCSIStager) AddTargetsInfoToMap( nvmetcpTargetInfo, err := identifiers.GetNVMETCPTargetsInfoFromStorage(client, volumeApplianceID) if err != nil { - log.Error("error unable to get NVMeTCP targets from array", err) + log.Errorf("error unable to get NVMeTCP targets from array %s", err.Error()) } for i, t := range nvmetcpTargetInfo { targetMap[fmt.Sprintf("%s%d", nvmeTCPPortalsKey, i)] = t.Portal diff --git a/pkg/node/stager_test.go b/pkg/node/stager_test.go index 5eed5273..56ee349e 100644 --- a/pkg/node/stager_test.go +++ b/pkg/node/stager_test.go @@ -26,17 +26,17 @@ import ( "reflect" "testing" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/mocks" "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/csmlog" "github.com/dell/gopowerstore" gopowerstoremock "github.com/dell/gopowerstore/mocks" + "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/gobrick" "github.com/dell/gofsutil" "github.com/golang/mock/gomock" - log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) @@ -64,7 +64,7 @@ func getValidPublishContext() map[string]string { } } -func getValidRemoteMetroPublishContext() map[string]string { +func getValidUniformMetroPublishContext() map[string]string { publishContext := getValidPublishContext() publishContext[identifiers.TargetMapRemoteLUNAddress] = validLUNID publishContext[identifiers.TargetMapRemoteDeviceWWN] = validDeviceWWN @@ -78,6 +78,22 @@ func getValidRemoteMetroPublishContext() map[string]string { return publishContext } +// getValidRemoteMetroPublishContext contains an empty local publish context +// and a populated remote publish context. +func getValidRemoteMetroPublishContext() map[string]string { + publishContext := make(map[string]string) + publishContext[identifiers.TargetMapRemoteLUNAddress] = validLUNID + publishContext[identifiers.TargetMapRemoteDeviceWWN] = validDeviceWWN + publishContext[identifiers.TargetMapRemoteISCSIPortalsPrefix+"0"] = validRemoteISCSIPortals[0] + publishContext[identifiers.TargetMapRemoteISCSIPortalsPrefix+"1"] = validRemoteISCSIPortals[1] + publishContext[identifiers.TargetMapRemoteISCSITargetsPrefix+"0"] = validRemoteISCSITargets[0] + publishContext[identifiers.TargetMapRemoteISCSITargetsPrefix+"1"] = validRemoteISCSITargets[1] + publishContext[identifiers.TargetMapRemoteFCWWPNPrefix+"0"] = validRemoteFCTargetsWWPN[0] + publishContext[identifiers.TargetMapRemoteFCWWPNPrefix+"1"] = validRemoteFCTargetsWWPN[1] + + return publishContext +} + func getCapabilityWithVoltypeAccessFstype(voltype, access, fstype string) *csi.VolumeCapability { // Construct the volume capability capability := new(csi.VolumeCapability) @@ -120,11 +136,16 @@ func scsiStageVolumeOK(util *mocks.UtilInterface, fs *mocks.FsInterface) { fs.On("GetUtil").Return(util) } +func scsiStageVolumeFail(util *mocks.UtilInterface, fs *mocks.FsInterface) { + util.On("BindMount", mock.Anything, "/dev", filepath.Join(nodeStagePrivateDir, validBaseVolumeID)).Return(nil) + fs.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, errors.New("mock staging failure")).Once() +} + func scsiStageRemoteMetroVolumeOK(util *mocks.UtilInterface, fs *mocks.FsInterface) { - util.On("BindMount", mock.Anything, "/dev", filepath.Join(nodeStagePrivateDir, validRemoteVolID)).Return(nil) + util.On("BindMount", mock.Anything, "/dev", filepath.Join(nodeStagePrivateDir, validBaseVolumeID)).Return(nil) fs.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fs.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) - fs.On("MkFileIdempotent", filepath.Join(nodeStagePrivateDir, validRemoteVolID)).Return(true, nil) + fs.On("MkFileIdempotent", filepath.Join(nodeStagePrivateDir, validBaseVolumeID)).Return(true, nil) fs.On("GetUtil").Return(util) } @@ -238,12 +259,12 @@ func TestSCSIStager_Stage(t *testing.T) { scsiStageVolumeOK(utilMock, fsMock) _, err := stager.Stage(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "block", "single-writer", "none"), - }, log.Fields{}, fsMock, validBaseVolumeID, false, clientMock) + }, filepath.Join(nodeStagePrivateDir, validBaseVolumeID), "node-1", csmlog.Fields{}, fsMock, validBaseVolumeID, false, clientMock) assert.Nil(t, err) }) @@ -270,12 +291,12 @@ func TestSCSIStager_Stage(t *testing.T) { scsiStageVolumeOK(utilMock, fsMock) _, err := stager.Stage(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "block", "single-writer", "none"), - }, log.Fields{}, fsMock, validBaseVolumeID, false, clientMock) + }, filepath.Join(nodeStagePrivateDir, validBaseVolumeID), "node-1", csmlog.Fields{}, fsMock, validBaseVolumeID, false, clientMock) assert.Nil(t, err) }) @@ -302,12 +323,12 @@ func TestSCSIStager_Stage(t *testing.T) { scsiStageVolumeOK(utilMock, fsMock) _, err := stager.Stage(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "block", "single-writer", "none"), - }, log.Fields{}, fsMock, validBaseVolumeID, false, clientMock) + }, filepath.Join(nodeStagePrivateDir, validBaseVolumeID), "node-1", csmlog.Fields{}, fsMock, validBaseVolumeID, false, clientMock) assert.Nil(t, err) }) @@ -346,12 +367,12 @@ func TestSCSIStager_Stage(t *testing.T) { scsiStageVolumeOK(utilMock, fsMock) _, err := stager.Stage(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "block", "single-writer", "none"), - }, log.Fields{}, fsMock, validBaseVolumeID, false, client) + }, filepath.Join(nodeStagePrivateDir, validBaseVolumeID), "node-1", csmlog.Fields{}, fsMock, validBaseVolumeID, false, client) assert.NotNil(t, err) assert.Contains(t, err.Error(), "unable to get targets for any protocol") }) @@ -393,12 +414,12 @@ func TestSCSIStager_Stage(t *testing.T) { scsiStageVolumeOK(utilMock, fsMock) _, err := stager.Stage(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "block", "single-writer", "none"), - }, log.Fields{}, fsMock, validBaseVolumeID, false, client) + }, filepath.Join(nodeStagePrivateDir, validBaseVolumeID), "node-1", csmlog.Fields{}, fsMock, validBaseVolumeID, false, client) assert.NotNil(t, err) assert.Contains(t, err.Error(), "NVMeFC Targets data must be in publish context") }) @@ -441,12 +462,12 @@ func TestSCSIStager_Stage(t *testing.T) { scsiStageVolumeOK(utilMock, fsMock) _, err := stager.Stage(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "block", "single-writer", "none"), - }, log.Fields{}, fsMock, validBaseVolumeID, false, client) + }, filepath.Join(nodeStagePrivateDir, validBaseVolumeID), "node-1", csmlog.Fields{}, fsMock, validBaseVolumeID, false, client) assert.NotNil(t, err) assert.Contains(t, err.Error(), "NVMeTCP Targets data must be in publish context") }) @@ -490,12 +511,12 @@ func TestSCSIStager_Stage(t *testing.T) { scsiStageVolumeOK(utilMock, fsMock) _, err := stager.Stage(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "block", "single-writer", "none"), - }, log.Fields{}, fsMock, validBaseVolumeID, false, client) + }, filepath.Join(nodeStagePrivateDir, validBaseVolumeID), "node-1", csmlog.Fields{}, fsMock, validBaseVolumeID, false, client) assert.NotNil(t, err) assert.Contains(t, err.Error(), "iscsiTargets data must be in publish context") }) @@ -537,17 +558,121 @@ func TestSCSIStager_Stage(t *testing.T) { scsiStageVolumeOK(utilMock, fsMock) _, err := stager.Stage(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "block", "single-writer", "none"), - }, log.Fields{}, fsMock, validBaseVolumeID, false, client) + }, filepath.Join(nodeStagePrivateDir, validBaseVolumeID), "node-1", csmlog.Fields{}, fsMock, validBaseVolumeID, false, client) assert.NotNil(t, err) assert.Contains(t, err.Error(), "fcTargets data must be in publish context") }) } +func TestSCSIStager_getLunAddressFromArray(t *testing.T) { + tests := []struct { + name string + getHostByNameFunc func(ctx context.Context, name string) (gopowerstore.Host, error) + getHostVolumeMappingFunc func(ctx context.Context, volumeID string) ([]gopowerstore.HostVolumeMapping, error) + volumeID string + nodeID string + want string + wantErr bool + }{ + { + name: "valid array and volume", + getHostByNameFunc: func(_ context.Context, _ string) (gopowerstore.Host, error) { + return gopowerstore.Host{ + ID: validHostID, + Initiators: []gopowerstore.InitiatorInstance{}, + Name: "host-name", + }, nil + }, + getHostVolumeMappingFunc: func(_ context.Context, _ string) ([]gopowerstore.HostVolumeMapping, error) { + return []gopowerstore.HostVolumeMapping{ + {HostID: validHostID, LogicalUnitNumber: validLUNIDINT}, + }, nil + }, + volumeID: "valid-volume-id", + nodeID: validHostID, + want: validLUNID, + wantErr: false, + }, + { + name: "invalid volume ID", + getHostByNameFunc: func(_ context.Context, _ string) (gopowerstore.Host, error) { + return gopowerstore.Host{ + ID: validHostID, + Initiators: []gopowerstore.InitiatorInstance{}, + Name: "host-name", + }, nil + }, + getHostVolumeMappingFunc: func(_ context.Context, _ string) ([]gopowerstore.HostVolumeMapping, error) { + return nil, errors.New("invalid volume ID") + }, + volumeID: "invalid-volume-id", + nodeID: "valid-node-id", + want: "", + wantErr: true, + }, + { + name: "host not found", + getHostByNameFunc: func(_ context.Context, _ string) (gopowerstore.Host, error) { + return gopowerstore.Host{}, errors.New("invalid host") + }, + getHostVolumeMappingFunc: func(_ context.Context, _ string) ([]gopowerstore.HostVolumeMapping, error) { + return []gopowerstore.HostVolumeMapping{ + {HostID: validHostID, LogicalUnitNumber: validLUNIDINT}, + }, nil + }, + volumeID: "valid-volume-id", + nodeID: "valid-node-id", + want: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientMock.ExpectedCalls = nil // reset previous expectations + + // Mock GetHostByName using closures + clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")). + Return(func(ctx context.Context, name string) gopowerstore.Host { + host, _ := tt.getHostByNameFunc(ctx, name) + return host + }, func(ctx context.Context, name string) error { + _, err := tt.getHostByNameFunc(ctx, name) + return err + }) + + // Mock GetHostVolumeMappingByVolumeID using closures + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, mock.AnythingOfType("string")). + Return(func(ctx context.Context, volumeID string) []gopowerstore.HostVolumeMapping { + mapping, _ := tt.getHostVolumeMappingFunc(ctx, volumeID) + return mapping + }, func(ctx context.Context, volumeID string) error { + _, err := tt.getHostVolumeMappingFunc(ctx, volumeID) + return err + }) + + // Call the method under test + got, err := getLunAddressFromArray(context.Background(), clientMock, tt.volumeID, tt.nodeID) + + // Validate error expectation + if (err != nil) != tt.wantErr { + t.Errorf("getLunAddressFromArray() error = %v, wantErr %v", err, tt.wantErr) + return + } + + // Validate result + if got != tt.want { + t.Errorf("getLunAddressFromArray() = %v, want %v", got, tt.want) + } + }) + } +} + func TestSCSIStager_AddTargetsInfoToMap(t *testing.T) { tests := []struct { name string diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go deleted file mode 100644 index 22d7f6df..00000000 --- a/pkg/provider/provider.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package provider - -import ( - "os" - "strings" - - "github.com/dell/csi-powerstore/v2/pkg/controller" - "github.com/dell/csi-powerstore/v2/pkg/identifiers" - "github.com/dell/csi-powerstore/v2/pkg/identity" - "github.com/dell/csi-powerstore/v2/pkg/node" - "github.com/dell/csi-powerstore/v2/pkg/service" - "github.com/dell/csm-sharednfs/nfs" - "github.com/dell/gocsi" - logrus "github.com/sirupsen/logrus" - "google.golang.org/grpc" -) - -// Log init -var Log = logrus.New() - -const namespaceFile = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" - -// New returns a new gocsi Storage Plug-in Provider. -func New(controllerSvc controller.Interface, identitySvc *identity.Service, nodeSvc node.Interface, interList []grpc.UnaryServerInterceptor) *gocsi.StoragePlugin { - svc := service.New() - service.PutControllerService(controllerSvc) - service.PutNodeService(nodeSvc) - nfssvc := nfs.New(identifiers.Name) - service.PutNfsService(nfssvc) - nfs.PutVcsiService(svc) - nfs.DriverName = identifiers.Name - - driverNamespace := os.Getenv(identifiers.EnvDriverNamespace) - if driverNamespace != "" { - Log.Infof("Reading driver namespace from env variable %s", identifiers.EnvDriverNamespace) - nfs.DriverNamespace = driverNamespace - } else { - // Read the namespace associated with the service account - namespaceData, err := os.ReadFile(namespaceFile) - if err == nil { - if driverNamespace = strings.TrimSpace(string(namespaceData)); len(driverNamespace) > 0 { - Log.Infof("Driver Namespace not set, reading from the associated service account") - nfs.DriverNamespace = driverNamespace - } - } - } - - nfs.NfsExportDirectory = os.Getenv(identifiers.EnvNFSExportDirectory) - if nfs.NfsExportDirectory == "" { - Log.Infof("NFS export directory not set. using default directory") - nfs.NfsExportDirectory = "/var/lib/dell/nfs" - } - Log.Infof("Setting nfsExportDirectory to %s", nfs.NfsExportDirectory) - return &gocsi.StoragePlugin{ - Controller: svc, - Identity: identitySvc, - Node: svc, - BeforeServe: svc.BeforeServe, - RegisterAdditionalServers: svc.RegisterAdditionalServers, - Interceptors: interList, - - EnvVars: []string{ - // Enable request validation - gocsi.EnvVarSpecReqValidation + "=true", - - // Enable serial volume access - gocsi.EnvVarSerialVolAccess + "=true", - }, - } -} diff --git a/pkg/provider/provider_test.go b/pkg/provider/provider_test.go deleted file mode 100644 index 64d1be74..00000000 --- a/pkg/provider/provider_test.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package provider - -import ( - "os" - "testing" - - "github.com/dell/csi-powerstore/v2/pkg/controller" - "github.com/dell/csi-powerstore/v2/pkg/identity" - "github.com/dell/csi-powerstore/v2/pkg/node" - "github.com/stretchr/testify/assert" -) - -func TestNew(t *testing.T) { - tests := []struct { - name string - setEnv func() - unsetEnv func() - createNamespaceFile func() - deleteNamespaceFile func() - }{ - { - name: "X_CSI_DRIVER_NAMESPACE environment variable is not set", - setEnv: func() {}, - unsetEnv: func() {}, - createNamespaceFile: func() { - err := os.MkdirAll("/var/run/secrets/kubernetes.io/serviceaccount", 0o755) - if err != nil { - t.Error(err) - } - file, err := os.Create(namespaceFile) - if err != nil { - t.Errorf("error creating file: %v", err) - t.Error(err) - } - defer func(file *os.File) { - err := file.Close() - if err != nil { - t.Error(err) - } - }(file) - - _, err = file.Write([]byte("powerstore")) - if err != nil { - t.Error(err) - } - }, - deleteNamespaceFile: func() { - err := os.Remove(namespaceFile) - if err != nil { - t.Error(err) - } - }, - }, - { - name: "X_CSI_DRIVER_NAMESPACE environment variable is set", - setEnv: func() { - err := os.Setenv("X_CSI_DRIVER_NAMESPACE", "powerstore") - if err != nil { - t.Error(err) - } - }, - unsetEnv: func() { - err := os.Unsetenv("X_CSI_DRIVER_NAMESPACE") - if err != nil { - t.Error(err) - } - }, - createNamespaceFile: func() {}, - deleteNamespaceFile: func() {}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - controllerSvc := controller.Service{} - identitySvc := identity.Service{} - nodeSvc := node.Service{} - tt.setEnv() - tt.createNamespaceFile() - p := New(&controllerSvc, &identitySvc, &nodeSvc, nil) - tt.deleteNamespaceFile() - tt.unsetEnv() - assert.NotNil(t, p) - }) - } -} diff --git a/pkg/service/controller.go b/pkg/service/controller.go deleted file mode 100644 index a7121159..00000000 --- a/pkg/service/controller.go +++ /dev/null @@ -1,224 +0,0 @@ -/* - * - * Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Package service provides CSI specification compatible controller service. -package service - -import ( - "context" - - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/dell/csm-sharednfs/nfs" - commonext "github.com/dell/dell-csi-extensions/common" - "github.com/sirupsen/logrus" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/metadata" - "google.golang.org/grpc/status" -) - -const ( - CsiNfsParameter = nfs.CsiNfsParameter - KeyNasName = "nasName" -) - -// CreateVolume creates either FileSystem or Volume on storage array. -func (s *service) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { - params := req.GetParameters() - if params[CsiNfsParameter] != "" { - params[CsiNfsParameter] = "RWX" - } - - if nfs.IsNFSStorageClass(params) { - if req.VolumeCapabilities[0].GetBlock() != nil { - return nil, status.Errorf(codes.InvalidArgument, "Block requested from Shared-NFS Volume") - } - return createHostBasedNFSVolume(ctx, req) - } - - return controllerSvc.CreateVolume(ctx, req) -} - -func createHostBasedNFSVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { - if contentSource := req.GetVolumeContentSource(); contentSource != nil { - // remove the nfs- prefix when cloning a host-based nfs volume, - // otherwise parsing the ID in the driver may fail to remove the prefix, - // and subsequent calls to the storage system API will fail because of a - // malformed volume UUID. - log.Info("creating host-based NFS volume from an existing source") - err := removeNFSPrefixFromSourceID(contentSource) - if err != nil { - return nil, err - } - } - - log.Infof("shared-nfs: RWX calling nfssvc.CreateVolume") - return nfssvc.CreateVolume(ctx, req) -} - -// removeNFSPrefixFromSourceID removes the nfs- prefix from a volume ID when a -// volume is specified as the volume content source of a CreateVolume request. -func removeNFSPrefixFromSourceID(source *csi.VolumeContentSource) error { - // if a volume is specified as volume content source, remove the nfs- prefix - if volume := source.GetVolume(); volume != nil { - if volume.VolumeId == "" { - return status.Error(codes.InvalidArgument, - "the volume ID of the volume to be cloned cannot be empty") - } - if !nfs.IsNFSVolumeID(volume.VolumeId) { - return status.Error(codes.InvalidArgument, - "the volume ID of the volume to be cloned must be of the host-based NFS type") - } - log.Infof("volume, %s, specified as volume content source", volume.VolumeId) - log.Debug("removing \"nfs-\" prefix from volume ID") - volume.VolumeId = nfs.ToArrayVolumeID(volume.VolumeId) - } - // snapshots should have been created without the "nfs-" prefix - if snapshot := source.GetSnapshot(); snapshot != nil { - log.Infof("snapshot, %s, specified as volume content source", snapshot.SnapshotId) - } - - return nil -} - -// DeleteVolume deletes either FileSystem or Volume from storage array. -func (s *service) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { - if nfs.IsNFSVolumeID(req.VolumeId) { - req.VolumeId = nfs.ToArrayVolumeID(req.VolumeId) - } - return controllerSvc.DeleteVolume(ctx, req) -} - -// ControllerPublishVolume prepares Volume/FileSystem to be consumed by node by attaching/allowing access to the host. -func (s *service) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { - log := getLogger(ctx) - volumeContext := req.GetVolumeContext() - if volumeContext != nil { - log.Debugf("VolumeContext:") - for key, value := range volumeContext { - log.Debugf(" [%s]=%s", key, value) - } - } - - // create publish context - publishContext := make(map[string]string) - publishContext[KeyNasName] = volumeContext[KeyNasName] - - csiVolID := req.GetVolumeId() - publishContext["volumeContextId"] = csiVolID - - if csiVolID == "" { - return nil, status.Error(codes.InvalidArgument, - "volume ID is required") - } - - if nfs.IsNFSVolumeID(csiVolID) { - log.Infof("shared-nfs: RWX calling nfssvc.ControllerPublishVolume") - return nfssvc.ControllerPublishVolume(ctx, req) - } - return controllerSvc.ControllerPublishVolume(ctx, req) -} - -// ControllerUnpublishVolume prepares Volume/FileSystem to be deleted by unattaching/disabling access to the host. -func (s *service) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { - log := getLogger(ctx) - if nfs.IsNFSVolumeID(req.GetVolumeId()) { - log.Info("shared-nfs: calling nfssrv.Controller.UnpublishVolume") - return nfssvc.ControllerUnpublishVolume(ctx, req) - } - return controllerSvc.ControllerUnpublishVolume(ctx, req) -} - -// ValidateVolumeCapabilities checks if capabilities found in request are supported by driver. -func (s *service) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { - if nfs.IsNFSVolumeID(req.VolumeId) { - req.VolumeId = nfs.ToArrayVolumeID(req.VolumeId) - } - return controllerSvc.ValidateVolumeCapabilities(ctx, req) -} - -// ListVolumes returns all accessible volumes from the storage array. -func (s *service) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) { - return controllerSvc.ListVolumes(ctx, req) -} - -// GetCapacity returns available capacity for a storage array. -func (s *service) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) { - return controllerSvc.GetCapacity(ctx, req) -} - -// ControllerGetCapabilities returns list of capabilities that are supported by the driver. -func (s *service) ControllerGetCapabilities(ctx context.Context, req *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { - return controllerSvc.ControllerGetCapabilities(ctx, req) -} - -// CreateSnapshot creates a snapshot of the Volume or FileSystem. -func (s *service) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { - if nfs.IsNFSVolumeID(req.SourceVolumeId) { - req.SourceVolumeId = nfs.ToArrayVolumeID(req.SourceVolumeId) - } - return controllerSvc.CreateSnapshot(ctx, req) -} - -// DeleteSnapshot deletes a snapshot of the Volume or FileSystem. -func (s *service) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { - return controllerSvc.DeleteSnapshot(ctx, req) -} - -// ListSnapshots list all accessible snapshots from the storage array. -func (s *service) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { - if nfs.IsNFSVolumeID(req.SourceVolumeId) { - req.SourceVolumeId = nfs.ToArrayVolumeID(req.SourceVolumeId) - } - return controllerSvc.ListSnapshots(ctx, req) -} - -// ControllerExpandVolume resizes Volume or FileSystem by increasing available volume capacity in the storage array. -func (s *service) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { - if nfs.IsNFSVolumeID(req.GetVolumeId()) { - req.VolumeId = nfs.ToArrayVolumeID(req.GetVolumeId()) - } - return controllerSvc.ControllerExpandVolume(ctx, req) -} - -// ControllerGetVolume fetch current information about a volume -func (s *service) ControllerGetVolume(ctx context.Context, req *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) { - if nfs.IsNFSVolumeID(req.VolumeId) { - req.VolumeId = nfs.ToArrayVolumeID(req.VolumeId) - } - return controllerSvc.ControllerGetVolume(ctx, req) -} - -// ProbeController probes the controller service -func (s *service) ProbeController(ctx context.Context, req *commonext.ProbeControllerRequest) (*commonext.ProbeControllerResponse, error) { - return controllerSvc.ProbeController(ctx, req) -} - -func getLogger(ctx context.Context) *logrus.Entry { - fields := logrus.Fields{} - headers, ok := metadata.FromIncomingContext(ctx) - if ok { - if req, ok := headers["csi.requestid"]; ok && len(req) > 0 { - fields["request_id"] = req[0] - } - } else { - if ctx.Value("request_id") != nil { - fields["request_id"] = ctx.Value("request_id").(string) - } - } - return logrus.WithFields(fields) -} diff --git a/pkg/service/controller_test.go b/pkg/service/controller_test.go deleted file mode 100644 index 3080795b..00000000 --- a/pkg/service/controller_test.go +++ /dev/null @@ -1,584 +0,0 @@ -/* - * - * Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package service - -import ( - "context" - "errors" - "testing" - - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/dell/csi-powerstore/v2/mocks" - "github.com/dell/csm-sharednfs/nfs" - nfsmock "github.com/dell/csm-sharednfs/nfs/mocks" - commonext "github.com/dell/dell-csi-extensions/common" - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "go.uber.org/mock/gomock" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func TestCreateVolume(t *testing.T) { - c := gomock.NewController(t) - svc := service{} - volumeUUID := uuid.New().String() - arrayGlobalID := "PS000000000001" - volumeHandle := volumeUUID + "/" + arrayGlobalID + "/scsi" - nfsVolumeHandle := nfs.CsiNfsPrefixDash + volumeHandle - - type args struct { - ctx context.Context - req *csi.CreateVolumeRequest - } - type testCase struct { - name string - args args - mockSetup func(mock *mocks.ControllerInterface, mockNode *mocks.MockInterface, mockNfs *nfsmock.MockService) - expectedErr error - } - - testCases := []testCase{ - { - name: "nfs volume", - args: args{ - ctx: context.Background(), - req: &csi.CreateVolumeRequest{ - Parameters: map[string]string{CsiNfsParameter: "RWX"}, - VolumeCapabilities: []*csi.VolumeCapability{ - { - AccessType: &csi.VolumeCapability_Mount{Mount: &csi.VolumeCapability_MountVolume{}}, - }, - }, - }, - }, - mockSetup: func(mockController *mocks.ControllerInterface, _ *mocks.MockInterface, mockNfs *nfsmock.MockService) { - mockController.On("CreateVolume", mock.Anything, mock.Anything).Return(&csi.CreateVolumeResponse{}, nil) - mockNfs.EXPECT().CreateVolume(gomock.Any(), gomock.Any()).Times(1).Return(&csi.CreateVolumeResponse{}, nil) - }, - expectedErr: nil, - }, - { - name: "normal volume", - args: args{ - ctx: context.Background(), - req: &csi.CreateVolumeRequest{ - VolumeCapabilities: []*csi.VolumeCapability{ - { - AccessType: &csi.VolumeCapability_Mount{Mount: &csi.VolumeCapability_MountVolume{}}, - }, - }, - }, - }, - mockSetup: func(mockController *mocks.ControllerInterface, _ *mocks.MockInterface, mockNfs *nfsmock.MockService) { - mockController.On("CreateVolume", mock.Anything, mock.Anything).Return(&csi.CreateVolumeResponse{}, nil) - mockNfs.EXPECT().CreateVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.CreateVolumeResponse{}, nil) - }, - expectedErr: nil, - }, - { - name: "clone a host-based NFS volume", - args: args{ - ctx: context.Background(), - req: &csi.CreateVolumeRequest{ - Parameters: map[string]string{CsiNfsParameter: "RWX"}, - // provide a host-based nfs volume as content source, signifying a clone request - VolumeCapabilities: []*csi.VolumeCapability{ - { - AccessType: &csi.VolumeCapability_Mount{Mount: &csi.VolumeCapability_MountVolume{}}, - }, - }, - VolumeContentSource: &csi.VolumeContentSource{ - Type: &csi.VolumeContentSource_Volume{ - Volume: &csi.VolumeContentSource_VolumeSource{ - VolumeId: nfsVolumeHandle, - }, - }, - }, - }, - }, - mockSetup: func(mockController *mocks.ControllerInterface, _ *mocks.MockInterface, mockNfs *nfsmock.MockService) { - mockController.On("CreateVolume", mock.Anything, mock.Anything).Return(&csi.CreateVolumeResponse{}, nil) - mockNfs.EXPECT().CreateVolume(gomock.Any(), gomock.Any()).Times(1).Return(&csi.CreateVolumeResponse{}, nil) - }, - expectedErr: nil, - }, - { - name: "clone a raw block volume with host-based NFS storage class", - args: args{ - ctx: context.Background(), - req: &csi.CreateVolumeRequest{ - // CsiNfsParameter denotes a host-based NFS storage class - Parameters: map[string]string{CsiNfsParameter: "RWX"}, - VolumeCapabilities: []*csi.VolumeCapability{ - { - AccessType: &csi.VolumeCapability_Mount{Mount: &csi.VolumeCapability_MountVolume{}}, - }, - }, - // provide a regular volume ID as content source - VolumeContentSource: &csi.VolumeContentSource{ - Type: &csi.VolumeContentSource_Volume{ - Volume: &csi.VolumeContentSource_VolumeSource{ - VolumeId: volumeHandle, - }, - }, - }, - }, - }, - mockSetup: func(_ *mocks.ControllerInterface, _ *mocks.MockInterface, _ *nfsmock.MockService) { - }, - expectedErr: status.Error(codes.InvalidArgument, - "the volume ID of the volume to be cloned must be of the host-based NFS type"), - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - mockController := new(mocks.ControllerInterface) - mockNode := mocks.NewMockInterface(c) - mockNfs := nfsmock.NewMockService(c) - - tc.mockSetup(mockController, mockNode, mockNfs) - - PutControllerService(mockController) - PutNodeService(mockNode) - PutNfsService(mockNfs) - - resp, err := svc.CreateVolume(tc.args.ctx, tc.args.req) - - if tc.expectedErr == nil { - assert.Nil(t, err) - assert.Empty(t, resp) - } else { - assert.Equal(t, tc.expectedErr, err) - } - }) - } -} - -func TestDeleteVolume(t *testing.T) { - c := gomock.NewController(t) - svc := service{} - ctx := context.Background() - mockController := new(mocks.ControllerInterface) - mockNode := mocks.NewMockInterface(c) - mockNfs := nfsmock.NewMockService(c) - - t.Run("nfs volume", func(t *testing.T) { - mockController.On("DeleteVolume", mock.Anything, mock.Anything).Return(&csi.DeleteVolumeResponse{}, nil) - mockNfs.EXPECT().DeleteVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.DeleteVolumeResponse{}, nil) - PutControllerService(mockController) - PutNodeService(mockNode) - PutNfsService(mockNfs) - - req := &csi.DeleteVolumeRequest{ - VolumeId: "nfs-123", - } - resp, err := svc.DeleteVolume(ctx, req) - assert.Nil(t, err) - assert.Empty(t, resp) - }) -} - -func TestProbeController(t *testing.T) { - svc := service{} - ctx := context.Background() - mockController := new(mocks.ControllerInterface) - - t.Run("success", func(t *testing.T) { - PutControllerService(mockController) - - req := &commonext.ProbeControllerRequest{} - mockController.On("ProbeController", mock.Anything, mock.Anything).Return(&commonext.ProbeControllerResponse{}, nil) - - resp, err := svc.ProbeController(ctx, req) - assert.Nil(t, err) - assert.Empty(t, resp) - }) -} - -func TestControllerGetVolume(t *testing.T) { - svc := service{} - ctx := context.Background() - mockController := new(mocks.ControllerInterface) - - t.Run("success", func(t *testing.T) { - PutControllerService(mockController) - - req := &csi.ControllerGetVolumeRequest{ - VolumeId: "nfs-12345", - } - mockController.On("ControllerGetVolume", mock.Anything, mock.Anything). - Return(&csi.ControllerGetVolumeResponse{}, nil) - - resp, err := svc.ControllerGetVolume(ctx, req) - assert.Nil(t, err) - assert.Empty(t, resp) - }) -} - -func TestControllerExpandVolume(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockController := new(mocks.ControllerInterface) - svc := service{} - req := &csi.ControllerExpandVolumeRequest{ - VolumeId: "nfs-volume", - } - mockController.On("ControllerExpandVolume", mock.Anything, req).Return(&csi.ControllerExpandVolumeResponse{}, nil) - PutControllerService(mockController) - resp, err := svc.ControllerExpandVolume(context.Background(), req) - assert.Nil(t, err) - assert.Empty(t, resp) -} - -func TestControllerListSnapshots(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockController := new(mocks.ControllerInterface) - svc := service{} - req := &csi.ListSnapshotsRequest{ - SourceVolumeId: "nfs-volume", - } - mockController.On("ListSnapshots", mock.Anything, req).Return(&csi.ListSnapshotsResponse{}, nil) - PutControllerService(mockController) - resp, err := svc.ListSnapshots(context.Background(), req) - assert.Nil(t, err) - assert.Empty(t, resp) -} - -func TestDeleteSnapshot(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockController := new(mocks.ControllerInterface) - svc := service{} - req := &csi.DeleteSnapshotRequest{ - SnapshotId: "nfs-snapshot", - } - mockController.On("DeleteSnapshot", mock.Anything, req).Return(&csi.DeleteSnapshotResponse{}, nil) - PutControllerService(mockController) - resp, err := svc.DeleteSnapshot(context.Background(), req) - assert.Nil(t, err) - assert.Empty(t, resp) -} - -func TestCreateSnapshot(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockController := new(mocks.ControllerInterface) - svc := service{} - req := &csi.CreateSnapshotRequest{ - SourceVolumeId: "nfs-volume", - Name: "snapshot-name", - } - mockController.On("CreateSnapshot", mock.Anything, req).Return(&csi.CreateSnapshotResponse{ - Snapshot: &csi.Snapshot{ - SnapshotId: "nfs-snapshot", - SourceVolumeId: "nfs-volume", - CreationTime: nil, - SizeBytes: int64(1024), - ReadyToUse: true, - }, - }, nil) - PutControllerService(mockController) - resp, err := svc.CreateSnapshot(context.Background(), req) - assert.Nil(t, err) - assert.NotEmpty(t, resp) - assert.Equal(t, "nfs-snapshot", resp.GetSnapshot().GetSnapshotId()) - assert.Equal(t, "nfs-volume", resp.GetSnapshot().GetSourceVolumeId()) - assert.Equal(t, int64(1024), resp.GetSnapshot().GetSizeBytes()) - assert.True(t, resp.GetSnapshot().GetReadyToUse()) -} - -func TestControllerGetCapabilities(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockController := new(mocks.ControllerInterface) - svc := service{} - req := &csi.ControllerGetCapabilitiesRequest{} - mockController.On("ControllerGetCapabilities", mock.Anything, req).Return(&csi.ControllerGetCapabilitiesResponse{}, nil) - PutControllerService(mockController) - resp, err := svc.ControllerGetCapabilities(context.Background(), req) - assert.Nil(t, err) - assert.Empty(t, resp) -} - -func TestControllerGetCapacity(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockController := new(mocks.ControllerInterface) - svc := service{} - req := &csi.GetCapacityRequest{} - mockController.On("GetCapacity", mock.Anything, req).Return(&csi.GetCapacityResponse{ - AvailableCapacity: int64(1024), - }, nil) - PutControllerService(mockController) - resp, err := svc.GetCapacity(context.Background(), req) - assert.Nil(t, err) - assert.NotEmpty(t, resp) - assert.Equal(t, int64(1024), resp.GetAvailableCapacity()) -} - -func TestControllerListVolumes(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockController := new(mocks.ControllerInterface) - svc := service{} - req := &csi.ListVolumesRequest{} - mockController.On("ListVolumes", mock.Anything, req).Return(&csi.ListVolumesResponse{ - Entries: []*csi.ListVolumesResponse_Entry{ - { - Volume: &csi.Volume{ - VolumeId: "nfs-123", - VolumeContext: map[string]string{"fsType": "nfs"}, - CapacityBytes: int64(1024), - }, - }, - }, - }, nil) - PutControllerService(mockController) - resp, err := svc.ListVolumes(context.Background(), req) - assert.Nil(t, err) - assert.NotEmpty(t, resp) - assert.Equal(t, "nfs-123", resp.GetEntries()[0].GetVolume().GetVolumeId()) - assert.Equal(t, "nfs", resp.GetEntries()[0].GetVolume().GetVolumeContext()["fsType"]) - assert.Equal(t, int64(1024), resp.GetEntries()[0].GetVolume().GetCapacityBytes()) -} - -func TestControllerValidateVolumeCapabilities(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockController := new(mocks.ControllerInterface) - svc := service{} - req := &csi.ValidateVolumeCapabilitiesRequest{ - VolumeId: "nfs-123", - } - mockController.On("ValidateVolumeCapabilities", mock.Anything, req).Return(&csi.ValidateVolumeCapabilitiesResponse{}, nil) - PutControllerService(mockController) - resp, err := svc.ValidateVolumeCapabilities(context.Background(), req) - assert.Nil(t, err) - assert.Empty(t, resp) -} - -func TestControllerUnpublishVolume(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockController := new(mocks.ControllerInterface) - mockNfs := nfsmock.NewMockService(ctrl) - svc := service{} - t.Run("nfs volume", func(t *testing.T) { - req := &csi.ControllerUnpublishVolumeRequest{ - VolumeId: "nfs-123", - NodeId: "node-123", - } - mockController.On("ControllerUnpublishVolume", mock.Anything, req).Return(&csi.ControllerUnpublishVolumeResponse{}, nil) - mockNfs.EXPECT().ControllerUnpublishVolume(gomock.Any(), req).AnyTimes().Return(&csi.ControllerUnpublishVolumeResponse{}, nil) - PutControllerService(mockController) - PutNfsService(mockNfs) - resp, err := svc.ControllerUnpublishVolume(context.Background(), req) - assert.Nil(t, err) - assert.Empty(t, resp) - }) - t.Run("normal volume", func(t *testing.T) { - req := &csi.ControllerUnpublishVolumeRequest{ - VolumeId: "vid-123", - NodeId: "node-123", - } - mockController.On("ControllerUnpublishVolume", mock.Anything, req).Return(&csi.ControllerUnpublishVolumeResponse{}, nil) - PutControllerService(mockController) - resp, err := svc.ControllerUnpublishVolume(context.Background(), req) - assert.Nil(t, err) - assert.Empty(t, resp) - }) -} - -func TestControllerPublishVolume(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - tests := []struct { - name string - req *csi.ControllerPublishVolumeRequest - expectResp *csi.ControllerPublishVolumeResponse - expectErr error - }{ - { - name: "empty volume id", - req: &csi.ControllerPublishVolumeRequest{ - VolumeContext: map[string]string{"fsType": "nfs"}, - VolumeId: "", - }, - expectErr: errors.New("volume ID is required"), - }, - { - name: "nfs volume", - req: &csi.ControllerPublishVolumeRequest{ - VolumeId: "nfs-123", - }, - expectResp: &csi.ControllerPublishVolumeResponse{}, - }, - { - name: "normal volume", - req: &csi.ControllerPublishVolumeRequest{ - VolumeId: "vid-123", - }, - expectResp: &csi.ControllerPublishVolumeResponse{}, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - mockController := new(mocks.ControllerInterface) - mockNfs := nfsmock.NewMockService(ctrl) - svc := service{} - - if tc.name == "nfs volume" { - mockController.On("ControllerPublishVolume", mock.Anything, tc.req).Return(tc.expectResp, tc.expectErr) - mockNfs.EXPECT().ControllerPublishVolume(gomock.Any(), tc.req).AnyTimes().Return(tc.expectResp, tc.expectErr) - - PutControllerService(mockController) - PutNfsService(mockNfs) - } else { - mockController.On("ControllerPublishVolume", mock.Anything, tc.req).Return(tc.expectResp, tc.expectErr) - - PutControllerService(mockController) - } - - resp, err := svc.ControllerPublishVolume(context.Background(), tc.req) - assert.Equal(t, tc.expectResp, resp) - if tc.expectErr != nil { - assert.Contains(t, err.Error(), tc.expectErr.Error()) - } - }) - } -} - -func Test_removeNFSPrefixFromSourceID(t *testing.T) { - arrayGlobalID := "PS000000000001" - volumeUUID := uuid.New().String() - volumeHandle := volumeUUID + "/" + arrayGlobalID + "/scsi" - nfsVolumeHandle := nfs.CsiNfsPrefixDash + volumeHandle - - type args struct { - source *csi.VolumeContentSource - } - tests := []struct { - name string - args args - expect *csi.VolumeContentSource - wantErr bool - errMsg string - }{ - { - name: "remove the nfs prefix from a host-based nfs source volume", - args: args{ - source: &csi.VolumeContentSource{ - Type: &csi.VolumeContentSource_Volume{ - Volume: &csi.VolumeContentSource_VolumeSource{ - VolumeId: nfsVolumeHandle, - }, - }, - }, - }, - expect: &csi.VolumeContentSource{ - Type: &csi.VolumeContentSource_Volume{ - Volume: &csi.VolumeContentSource_VolumeSource{ - VolumeId: volumeHandle, - }, - }, - }, - wantErr: false, - }, - { - name: "snapshots are not affected", - args: args{ - source: &csi.VolumeContentSource{ - Type: &csi.VolumeContentSource_Snapshot{ - Snapshot: &csi.VolumeContentSource_SnapshotSource{ - SnapshotId: volumeUUID, - }, - }, - }, - }, - expect: &csi.VolumeContentSource{ - Type: &csi.VolumeContentSource_Snapshot{ - Snapshot: &csi.VolumeContentSource_SnapshotSource{ - SnapshotId: volumeUUID, - }, - }, - }, - wantErr: false, - }, - { - name: "source volume ID is empty", - args: args{ - source: &csi.VolumeContentSource{ - Type: &csi.VolumeContentSource_Volume{ - Volume: &csi.VolumeContentSource_VolumeSource{ - VolumeId: "", - }, - }, - }, - }, - expect: &csi.VolumeContentSource{ - Type: &csi.VolumeContentSource_Volume{ - Volume: &csi.VolumeContentSource_VolumeSource{ - VolumeId: "", - }, - }, - }, - wantErr: true, - errMsg: "the volume ID of the volume to be cloned cannot be empty", - }, - { - name: "source volume is not a host-based NFS volume", - args: args{ - source: &csi.VolumeContentSource{ - Type: &csi.VolumeContentSource_Volume{ - Volume: &csi.VolumeContentSource_VolumeSource{ - VolumeId: volumeHandle, - }, - }, - }, - }, - expect: &csi.VolumeContentSource{ - Type: &csi.VolumeContentSource_Volume{ - Volume: &csi.VolumeContentSource_VolumeSource{ - VolumeId: volumeHandle, - }, - }, - }, - wantErr: true, - errMsg: "the volume ID of the volume to be cloned must be of the host-based NFS type", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := removeNFSPrefixFromSourceID(tt.args.source) - if (err != nil) != tt.wantErr { - t.Errorf("removeNFSPrefixFromSourceID() error = %v, wantErr %v", err, tt.wantErr) - } - if tt.wantErr { - assert.Contains(t, err.Error(), tt.errMsg) - } - assert.Equal(t, *tt.expect, *tt.args.source) - }) - } -} diff --git a/pkg/service/identity.go b/pkg/service/identity.go deleted file mode 100644 index d2730c9e..00000000 --- a/pkg/service/identity.go +++ /dev/null @@ -1,42 +0,0 @@ -/* - * - * Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Package identity provides CSI specification compatible identity service. -package service - -import ( - "context" - "fmt" - - "github.com/container-storage-interface/spec/lib/go/csi" -) - -// GetPluginInfo returns general information about plugin (driver) such as name, version and manifest -func (s *service) GetPluginInfo(_ context.Context, _ *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { - return &csi.GetPluginInfoResponse{}, fmt.Errorf("not implemented") -} - -// GetPluginCapabilities returns capabilities that are supported by the driver -func (s *service) GetPluginCapabilities(_ context.Context, _ *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { - return &csi.GetPluginCapabilitiesResponse{}, fmt.Errorf("not implemented") -} - -// Probe returns current state of the driver and if it is ready to receive requests -func (s *service) Probe(_ context.Context, _ *csi.ProbeRequest) (*csi.ProbeResponse, error) { - return &csi.ProbeResponse{}, fmt.Errorf("not implemented") -} diff --git a/pkg/service/identity_test.go b/pkg/service/identity_test.go deleted file mode 100644 index f5cbdce6..00000000 --- a/pkg/service/identity_test.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package service - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestGetPluginInfo(t *testing.T) { - svc := service{} - resp, err := svc.GetPluginInfo(context.Background(), nil) - assert.Empty(t, resp) - assert.Equal(t, err.Error(), "not implemented") -} - -func TestGetPluginCapabilities(t *testing.T) { - svc := service{} - resp, err := svc.GetPluginCapabilities(context.Background(), nil) - assert.Empty(t, resp) - assert.Equal(t, err.Error(), "not implemented") -} - -func TestProbe(t *testing.T) { - svc := service{} - resp, err := svc.Probe(context.Background(), nil) - assert.Empty(t, resp) - assert.Equal(t, err.Error(), "not implemented") -} diff --git a/pkg/service/node.go b/pkg/service/node.go deleted file mode 100644 index 9c4f13bd..00000000 --- a/pkg/service/node.go +++ /dev/null @@ -1,203 +0,0 @@ -/* - * - * Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Package node provides CSI specification compatible node service. -package service - -import ( - "context" - "fmt" - "os" - "path" - "path/filepath" - - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/dell/csm-sharednfs/nfs" -) - -var osRemove = os.RemoveAll - -// NodeStageVolume prepares volume to be consumed by node publish by connecting volume to the node -func (s *service) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { - if nfs.IsNFSVolumeID(req.GetVolumeId()) { - return nfssvc.NodeStageVolume(ctx, req) - } - return nodeSvc.NodeStageVolume(ctx, req) -} - -// NodeUnstageVolume reverses steps done in NodeStage by disconnecting volume from the node -func (s *service) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { - if nfs.IsNFSVolumeID(req.GetVolumeId()) { - return nfssvc.NodeUnstageVolume(ctx, req) - } - return nodeSvc.NodeUnstageVolume(ctx, req) -} - -// NodePublishVolume publishes volume to the node by mounting it to the target path -func (s *service) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { - if nfs.IsNFSVolumeID(req.GetVolumeId()) { - log.Infof("shared-nfs: RWX calling nfssvc.NodePublishVolume") - return nfssvc.NodePublishVolume(ctx, req) - } - return nodeSvc.NodePublishVolume(ctx, req) -} - -// NodeUnpublishVolume unpublishes volume from the node by unmounting it from the target path -func (s *service) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { - if nfs.IsNFSVolumeID(req.GetVolumeId()) { - log.Infof("shared-nfs: RWX calling nfssvc.NodeUnpublishVolume") - return nfssvc.NodeUnpublishVolume(ctx, req) - } - - return nodeSvc.NodeUnpublishVolume(ctx, req) -} - -// NodeGetVolumeStats returns volume usage stats -func (s *service) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { - if nfs.IsNFSVolumeID(req.VolumeId) { - req.VolumeId = nfs.ToArrayVolumeID(req.VolumeId) - } - return nodeSvc.NodeGetVolumeStats(ctx, req) -} - -// NodeExpandVolume expands the volume by re-scanning and resizes filesystem if needed -func (s *service) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) { - log.Infof("NodeExpandVolume called req %v", req) - return nodeSvc.NodeExpandVolume(ctx, req) -} - -// NodeGetCapabilities returns supported features by the node service -func (s *service) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { - return nodeSvc.NodeGetCapabilities(ctx, req) -} - -// NodeGetInfo returns id of the node and topology constraints -func (s *service) NodeGetInfo(ctx context.Context, _ *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { - return nodeSvc.NodeGetInfo(ctx, nil) -} - -func (s *service) MountVolume(ctx context.Context, volumeID, fsType, nfsExportDirectory string, publishContext map[string]string) (string, error) { - log.Infof("MountVolume called volumeId %s", volumeID) - if volumeID == "" { - return "", fmt.Errorf("MountVolume: volumeId was empty") - } - - if nfsExportDirectory == "" { - nfsExportDirectory = nfs.NfsExportDirectory - } - // the Stage volume will create a file and mount the device directly to the file - staging := path.Join(nfsExportDirectory, publishContext[nfs.ServiceName]+"-dev") - target := path.Join(nfsExportDirectory, publishContext[nfs.ServiceName]) - - nodeStageReq := &csi.NodeStageVolumeRequest{ - VolumeId: volumeID, - VolumeCapability: &csi.VolumeCapability{ - AccessType: &csi.VolumeCapability_Mount{ - Mount: &csi.VolumeCapability_MountVolume{ - MountFlags: []string{"rw"}, - }, - }, - AccessMode: &csi.VolumeCapability_AccessMode{ - Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER, - }, - }, - StagingTargetPath: staging, - PublishContext: publishContext, - } - - log.Infof("MountVolume calling NodeStageVolume: %+v", nodeStageReq) - _, err := nodeSvc.NodeStageVolume(ctx, nodeStageReq) - if err != nil { - return "", fmt.Errorf("MountVolume: could not stage volume volumeId %s: %s", volumeID, err) - } - - nodePubReq := &csi.NodePublishVolumeRequest{ - VolumeId: volumeID, - VolumeCapability: &csi.VolumeCapability{ - AccessType: &csi.VolumeCapability_Mount{ - Mount: &csi.VolumeCapability_MountVolume{ - MountFlags: []string{"rw"}, - FsType: fsType, - }, - }, - AccessMode: &csi.VolumeCapability_AccessMode{ - Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_SINGLE_WRITER, - }, - }, - PublishContext: publishContext, - StagingTargetPath: staging, - TargetPath: target, - Readonly: false, - } - - log.Infof("MountVolume calling NodePublishVolume %+v", nodePubReq) - _, err = nodeSvc.NodePublishVolume(ctx, nodePubReq) - if err != nil { - return "", fmt.Errorf("MountVolume: could not publish volume volumeId %s", volumeID) - } - log.Infof("MountVolume ALL GOOD volume %s mounted to target %s", volumeID, target) - return target, nil -} - -func (s *service) UnmountVolume(ctx context.Context, volumeID, exportPath string, publishContext map[string]string) error { - staging := path.Join(exportPath, publishContext[nfs.ServiceName]+"-dev") - target := path.Join(exportPath, publishContext[nfs.ServiceName]) - - target = filepath.Clean(target) - staging = filepath.Clean(staging) - - nodeUnpublishReq := &csi.NodeUnpublishVolumeRequest{ - VolumeId: volumeID, - TargetPath: target, - } - - log.Infof("[UnmountVolume] Target: %s", target) - // Unpublish the target directory for sharedNFS first. This will properly clean up and /dev/sdX entries associated to the mount. - _, err := nodeSvc.NodeUnpublishVolume(ctx, nodeUnpublishReq) - if err != nil { - return fmt.Errorf("[UnmountVolume] unpublish failed for %s: %e", target, err) - } - - nodeUnstageReq := &csi.NodeUnstageVolumeRequest{ - VolumeId: volumeID, - StagingTargetPath: staging, - } - - // Unmount the staging directory for sharedNFS (-dev) to clean up anything related to the mount. - log.Infof("[UnmountVolume] Staging: %s", staging) - _, err = nodeSvc.NodeUnstageVolume(ctx, nodeUnstageReq) - if err != nil { - return fmt.Errorf("[UnmountVolume] unstage failed for %s: %e", staging, err) - } - - // After successfully unmounting/unstaging both target and staging, remove both. - err = osRemove(target) - if err != nil && !os.IsNotExist(err) { - log.Errorf("[UnmountVolume] could not remove target path %s: %s", target, err.Error()) - return err - } - - err = osRemove(staging) - if err != nil && !os.IsNotExist(err) { - log.Errorf("[UnmountVolume] could not remove staging path %s: %s", staging, err.Error()) - return err - } - - log.Infof("[UnmountVolume] Successfully unmounted volume %s", volumeID) - return nil -} diff --git a/pkg/service/node_test.go b/pkg/service/node_test.go deleted file mode 100644 index 39fe98cf..00000000 --- a/pkg/service/node_test.go +++ /dev/null @@ -1,367 +0,0 @@ -/* - * - * Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package service - -import ( - "context" - "errors" - "fmt" - "strings" - "testing" - - csi "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/dell/csi-powerstore/v2/mocks" - nfsmock "github.com/dell/csm-sharednfs/nfs/mocks" - "github.com/stretchr/testify/assert" - "go.uber.org/mock/gomock" -) - -func TestNodeGetInfo(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockNode := mocks.NewMockInterface(ctrl) - svc := service{} - mockNode.EXPECT().NodeGetInfo(gomock.Any(), gomock.Any()).Return(&csi.NodeGetInfoResponse{}, nil) - PutNodeService(mockNode) - req := &csi.NodeGetInfoRequest{} - resp, err := svc.NodeGetInfo(context.Background(), req) - assert.Nil(t, err) - assert.Empty(t, resp) -} - -func TestNodeGetCapabilities(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockNode := mocks.NewMockInterface(ctrl) - svc := service{} - mockNode.EXPECT().NodeGetCapabilities(gomock.Any(), gomock.Any()).Return(&csi.NodeGetCapabilitiesResponse{}, nil) - PutNodeService(mockNode) - req := &csi.NodeGetCapabilitiesRequest{} - resp, err := svc.NodeGetCapabilities(context.Background(), req) - assert.Nil(t, err) - assert.Empty(t, resp) -} - -func TestNodeExpandVolume(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockNode := mocks.NewMockInterface(ctrl) - svc := service{} - mockNode.EXPECT().NodeExpandVolume(gomock.Any(), gomock.Any()).Return(&csi.NodeExpandVolumeResponse{}, nil) - PutNodeService(mockNode) - req := &csi.NodeExpandVolumeRequest{} - resp, err := svc.NodeExpandVolume(context.Background(), req) - assert.Nil(t, err) - assert.Empty(t, resp) -} - -func TestNodeGetVolumeStats(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockNode := mocks.NewMockInterface(ctrl) - svc := service{} - mockNode.EXPECT().NodeGetVolumeStats(gomock.Any(), gomock.Any()).Return(&csi.NodeGetVolumeStatsResponse{}, nil) - PutNodeService(mockNode) - req := &csi.NodeGetVolumeStatsRequest{ - VolumeId: "nfs-123", - } - resp, err := svc.NodeGetVolumeStats(context.Background(), req) - assert.Nil(t, err) - assert.Empty(t, resp) -} - -func TestNodeUnpublishVolume(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockNode := mocks.NewMockInterface(ctrl) - svc := service{} - mockNode.EXPECT().NodeUnpublishVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodeUnpublishVolumeResponse{}, nil) - PutNodeService(mockNode) - - t.Run("nfs volume", func(t *testing.T) { - mockNfs := nfsmock.NewMockService(ctrl) - mockNfs.EXPECT().NodeUnpublishVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodeUnpublishVolumeResponse{}, nil) - PutNfsService(mockNfs) - req := &csi.NodeUnpublishVolumeRequest{ - VolumeId: "nfs-123", - } - resp, err := svc.NodeUnpublishVolume(context.Background(), req) - assert.Nil(t, err) - assert.Empty(t, resp) - }) - - t.Run("normal volume", func(t *testing.T) { - req := &csi.NodeUnpublishVolumeRequest{ - VolumeId: "123", - } - resp, err := svc.NodeUnpublishVolume(context.Background(), req) - assert.Nil(t, err) - assert.Empty(t, resp) - }) -} - -func TestNodePublishVolume(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockNode := mocks.NewMockInterface(ctrl) - svc := service{} - mockNode.EXPECT().NodePublishVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodePublishVolumeResponse{}, nil) - PutNodeService(mockNode) - - t.Run("nfs volume", func(t *testing.T) { - mockNfs := nfsmock.NewMockService(ctrl) - mockNfs.EXPECT().NodePublishVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodePublishVolumeResponse{}, nil) - PutNfsService(mockNfs) - req := &csi.NodePublishVolumeRequest{ - VolumeId: "nfs-123", - } - resp, err := svc.NodePublishVolume(context.Background(), req) - assert.Nil(t, err) - assert.Empty(t, resp) - }) - - t.Run("normal volume", func(t *testing.T) { - req := &csi.NodePublishVolumeRequest{ - VolumeId: "123", - } - resp, err := svc.NodePublishVolume(context.Background(), req) - assert.Nil(t, err) - assert.Empty(t, resp) - }) -} - -func TestNodeUnstageVolume(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockNode := mocks.NewMockInterface(ctrl) - svc := service{} - mockNode.EXPECT().NodeUnstageVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodeUnstageVolumeResponse{ - XXX_sizecache: 10, - }, nil) - PutNodeService(mockNode) - t.Run("nfs volume", func(t *testing.T) { - req := &csi.NodeUnstageVolumeRequest{ - VolumeId: "nfs-aaaa-bbbb-cccc-dddd-eeeeeeeeeeee", - } - mockNFSSvc := nfsmock.NewMockService(ctrl) - mockNFSSvc.EXPECT().NodeUnstageVolume(gomock.Any(), req).AnyTimes().Return(&csi.NodeUnstageVolumeResponse{}, nil) - PutNfsService(mockNFSSvc) - - resp, err := svc.NodeUnstageVolume(context.Background(), req) - assert.Nil(t, err) - assert.Empty(t, resp) - }) - - t.Run("normal volume", func(t *testing.T) { - req := &csi.NodeUnstageVolumeRequest{} - resp, err := svc.NodeUnstageVolume(context.Background(), req) - assert.Nil(t, err) - assert.Equal(t, resp.XXX_sizecache, int32(10)) - }) -} - -func TestNodeStageVolume(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockNode := mocks.NewMockInterface(ctrl) - svc := service{} - mockNode.EXPECT().NodeStageVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodeStageVolumeResponse{ - XXX_sizecache: 10, - }, nil) - PutNodeService(mockNode) - t.Run("nfs volume", func(t *testing.T) { - req := &csi.NodeStageVolumeRequest{ - VolumeId: "nfs-aaaa-bbbb-cccc-dddd-eeeeeeeeeeee", - } - mockNFSSvc := nfsmock.NewMockService(ctrl) - mockNFSSvc.EXPECT().NodeStageVolume(gomock.Any(), req).AnyTimes().Return(&csi.NodeStageVolumeResponse{}, nil) - PutNfsService(mockNFSSvc) - - resp, err := svc.NodeStageVolume(context.Background(), req) - assert.Nil(t, err) - assert.Empty(t, resp) - }) - - t.Run("normal volume", func(t *testing.T) { - req := &csi.NodeStageVolumeRequest{ - VolumeId: "123", - } - resp, err := svc.NodeStageVolume(context.Background(), req) - assert.Nil(t, err) - assert.NotEmpty(t, resp) - }) -} - -func TestMountVolume(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - svc := service{} - ctx := context.Background() - - t.Run("no volume id", func(t *testing.T) { - resp, err := svc.MountVolume(ctx, "", "", "", map[string]string{}) - assert.Empty(t, resp) - assert.Equal(t, err.Error(), "MountVolume: volumeId was empty") - }) - - t.Run("success", func(t *testing.T) { - mockNode := mocks.NewMockInterface(ctrl) - mockNfs := nfsmock.NewMockService(ctrl) - mockNode.EXPECT().NodePublishVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodePublishVolumeResponse{}, nil) - mockNfs.EXPECT().NodePublishVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodePublishVolumeResponse{}, nil) - mockNode.EXPECT().NodeStageVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodeStageVolumeResponse{}, nil) - mockNfs.EXPECT().NodeStageVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodeStageVolumeResponse{}, nil) - PutNfsService(mockNfs) - PutNodeService(mockNode) - resp, err := svc.MountVolume(ctx, "123", "", "test", map[string]string{}) - assert.Nil(t, err) - assert.Contains(t, resp, "test") - resp, err = svc.MountVolume(ctx, "123", "", "", map[string]string{}) - assert.Nil(t, err) - assert.Empty(t, resp) - }) - - t.Run("stage error", func(t *testing.T) { - mockNode := mocks.NewMockInterface(ctrl) - mockNfs := nfsmock.NewMockService(ctrl) - mockNode.EXPECT().NodePublishVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodePublishVolumeResponse{}, nil) - mockNfs.EXPECT().NodePublishVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodePublishVolumeResponse{}, nil) - mockNode.EXPECT().NodeStageVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodeStageVolumeResponse{}, errors.New("stage error")) - mockNfs.EXPECT().NodeStageVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodeStageVolumeResponse{}, errors.New("stage error")) - PutNfsService(mockNfs) - PutNodeService(mockNode) - resp, err := svc.MountVolume(ctx, "123", "", "", map[string]string{}) - assert.Empty(t, resp) - assert.Contains(t, err.Error(), "stage error") - }) - - t.Run("publish error", func(t *testing.T) { - mockNode := mocks.NewMockInterface(ctrl) - mockNfs := nfsmock.NewMockService(ctrl) - mockNode.EXPECT().NodePublishVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodePublishVolumeResponse{}, errors.New("publish error")) - mockNfs.EXPECT().NodePublishVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodePublishVolumeResponse{}, errors.New("publish error")) - mockNode.EXPECT().NodeStageVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodeStageVolumeResponse{}, nil) - mockNfs.EXPECT().NodeStageVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodeStageVolumeResponse{}, nil) - PutNfsService(mockNfs) - PutNodeService(mockNode) - resp, err := svc.MountVolume(ctx, "123", "", "", map[string]string{}) - assert.Empty(t, resp) - assert.Contains(t, err.Error(), "could not publish volume") - }) -} - -func TestUnmountVolume(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - ctx := context.Background() - - t.Run("fail: unable to unpublish volume", func(t *testing.T) { - mockNode := mocks.NewMockInterface(ctrl) - oldRemove := osRemove - - defer func() { - osRemove = oldRemove - }() - - osRemove = func(_ string) error { - return nil - } - - svc := service{} - - mockNode.EXPECT().NodeUnpublishVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(nil, fmt.Errorf("failed to unstage volume")) - PutNodeService(mockNode) - - err := svc.UnmountVolume(ctx, "", "/var/lib/dell/nfs/", map[string]string{ - "ServiceName": "myNfsMount", - }) - assert.NotNil(t, err) - }) - - t.Run("success: unmount volume", func(t *testing.T) { - mockNode := mocks.NewMockInterface(ctrl) - oldRemove := osRemove - defer func() { - osRemove = oldRemove - }() - - osRemove = func(_ string) error { - return nil - } - - mockNode.EXPECT().NodeUnpublishVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodeUnpublishVolumeResponse{}, nil) - mockNode.EXPECT().NodeUnstageVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodeUnstageVolumeResponse{}, nil) - PutNodeService(mockNode) - - svc := service{} - - err := svc.UnmountVolume(ctx, "123", "/var/lib/dell/nfs/", map[string]string{ - "ServiceName": "myNfsMount", - }) - assert.Nil(t, err) - }) - - t.Run("fail: osRemove error of target", func(t *testing.T) { - mockNode := mocks.NewMockInterface(ctrl) - oldRemove := osRemove - defer func() { - osRemove = oldRemove - }() - - osRemove = func(_ string) error { - return errors.New("unable to remove target") - } - - mockNode.EXPECT().NodeUnpublishVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodeUnpublishVolumeResponse{}, nil) - mockNode.EXPECT().NodeUnstageVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodeUnstageVolumeResponse{}, nil) - PutNodeService(mockNode) - - svc := service{} - - err := svc.UnmountVolume(ctx, "123", "/var/lib/dell/nfs/", map[string]string{ - "ServiceName": "myNfsMount", - }) - assert.Contains(t, err.Error(), "unable to remove target") - }) - - t.Run("fail: osRemove error of staging", func(t *testing.T) { - mockNode := mocks.NewMockInterface(ctrl) - oldRemove := osRemove - defer func() { - osRemove = oldRemove - }() - - osRemove = func(file string) error { - if strings.Contains(file, "-dev") { - return errors.New("unable to remove staging") - } - - return nil - } - - mockNode.EXPECT().NodeUnpublishVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodeUnpublishVolumeResponse{}, nil) - mockNode.EXPECT().NodeUnstageVolume(gomock.Any(), gomock.Any()).AnyTimes().Return(&csi.NodeUnstageVolumeResponse{}, nil) - PutNodeService(mockNode) - - svc := service{} - - err := svc.UnmountVolume(ctx, "123", "/var/lib/dell/nfs/", map[string]string{ - "ServiceName": "myNfsMount", - }) - assert.Contains(t, err.Error(), "unable to remove staging") - }) -} diff --git a/pkg/service/service.go b/pkg/service/service.go deleted file mode 100644 index 29d6f615..00000000 --- a/pkg/service/service.go +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package service - -import ( - "context" - "fmt" - "net" - "os" - "strings" - "sync" - "time" - - "github.com/dell/csi-powerstore/v2/pkg/controller" - "github.com/dell/csi-powerstore/v2/pkg/identifiers" - "github.com/dell/csi-powerstore/v2/pkg/node" - "github.com/dell/csm-sharednfs/nfs" - "github.com/dell/gocsi" - csictx "github.com/dell/gocsi/context" - "github.com/fsnotify/fsnotify" - "github.com/sirupsen/logrus" - "github.com/spf13/viper" - "google.golang.org/grpc" -) - -// Log controlls the logger -// give default value, will be overwritten by configmap -var log = logrus.New() - -var ( - // DriverConfig driver config - DriverConfig string - // DriverSecret driver secret - DriverSecret string - // Name of the driver - controllerSvc controller.Interface - nodeSvc node.Interface - nfssvc nfs.Service - - mx = sync.Mutex{} -) - -type service struct { - mode string -} - -func New() nfs.Service { - return &service{} -} - -func (s *service) BeforeServe(ctx context.Context, _ *gocsi.StoragePlugin, _ net.Listener) error { - log.Info("-----Inside Before Serve-----") - // Get the SP's operating mode. - s.mode = csictx.Getenv(ctx, gocsi.EnvVarMode) - log.Info("Driver Mode:", s.mode) - // TODO: add nfs code here - nodeName := os.Getenv(identifiers.EnvKubeNodeName) - if nodeName == "" { - nodeName = os.Getenv("KUBE_NODE_NAME") - } - - if nodeName == "" { - nodeName = os.Getenv("X_CSI_NODE_NAME") - } - - if s.mode == "node" { - nodeRoot := os.Getenv(identifiers.EnvNodeChrootPath) - if nodeRoot == "" { - return fmt.Errorf("X_CSI_POWERSTORE_NODE_CHROOT_PATH environment variable not set") - } - nfs.NodeRoot = nodeRoot - } - - log.Infof("Setting node name env to %s for NFS", nodeName) - err := os.Setenv("X_CSI_NODE_NAME", nodeName) - if err != nil { - log.Errorf("failed to set env X_CSI_NODE_NAME. err: %s", err.Error()) - return err - } - - // The block is commented out for performance issue caused by sharednfs even when it's disabled. - // Remove the comment when enabling sharednfs feature. - /* - err = nfssvc.BeforeServe(ctx, sp, lis) - if err != nil { - log.Errorf("unable to start up nfsserver: %s", err.Error()) - } - */ - return nil -} - -func (s *service) RegisterAdditionalServers(server *grpc.Server) { - controllerSvc.RegisterAdditionalServers(server) -} - -func (s *service) ProcessMapSecretChange() error { - // Update dynamic config params - vc := viper.New() - vc.AutomaticEnv() - paramsPath, ok := csictx.LookupEnv(context.Background(), identifiers.EnvConfigParamsFilePath) - if !ok { - log.Warnf("config path X_CSI_POWERSTORE_CONFIG_PARAMS_PATH is not specified") - } - log.WithField("file", paramsPath).Info("driver configuration file ") - vc.SetConfigFile(paramsPath) - vc.SetConfigType("yaml") - if err := vc.ReadInConfig(); err != nil { - log.WithError(err).Error("unable to read config file, using default values") - } - - vc.WatchConfig() - vc.OnConfigChange(func(_ fsnotify.Event) { - // Putting in mutex to allow tests to pass with race flag - mx.Lock() - defer mx.Unlock() - log.WithField("file", paramsPath).Info("log configuration file changed") - updateDriverConfigParams(vc) - }) - - updateDriverConfigParams(vc) - - // If we don't set this env gocsi will overwrite log level with default Info level - err := os.Setenv(gocsi.EnvVarLogLevel, log.GetLevel().String()) - if err != nil { - log.WithError(err).Errorf("unable to set env variable %s", gocsi.EnvVarDebug) - } - - return err -} - -func updateDriverConfigParams(v *viper.Viper) { - logLevelParam := "CSI_LOG_LEVEL" - logFormatParam := "CSI_LOG_FORMAT" - logFormat := strings.ToLower(v.GetString(logFormatParam)) - fmt.Printf("Read CSI_LOG_FORMAT from log configuration file, format: %s\n", logFormat) - - // Use JSON logger as default - if !strings.EqualFold(logFormat, "text") { - log.SetFormatter(&logrus.JSONFormatter{ - TimestampFormat: time.RFC3339Nano, - }) - } else { - log.SetFormatter(&logrus.TextFormatter{}) - } - - level := logrus.DebugLevel - if v.IsSet(logLevelParam) { - logLevel := v.GetString(logLevelParam) - if logLevel != "" { - logLevel = strings.ToLower(logLevel) - fmt.Printf("Read CSI_LOG_LEVEL from log configuration file, level: %s\n", logLevel) - var err error - - l, err := logrus.ParseLevel(logLevel) - if err != nil { - log.WithError(err).Errorf("LOG_LEVEL %s value not recognized, setting to default error: %s ", logLevel, err.Error()) - } else { - level = l - } - } - } - log.SetLevel(level) -} - -// VolumeIDToArrayID returns the array ID for a given volume. -// Example: abc-123 returns abc -func (s *service) VolumeIDToArrayID(volumeID string) string { - if volumeID == "" { - return "" - } - fields := strings.Split(volumeID, "-") - return fields[0] -} - -func PutNfsService(nfs nfs.Service) { - nfssvc = nfs -} - -func PutControllerService(ctlSvc controller.Interface) { - controllerSvc = ctlSvc -} - -func PutNodeService(nsSvc node.Interface) { - nodeSvc = nsSvc -} diff --git a/pkg/service/service_test.go b/pkg/service/service_test.go deleted file mode 100644 index b4d67d81..00000000 --- a/pkg/service/service_test.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package service - -import ( - "context" - "fmt" - "os" - "strings" - "testing" - - "github.com/dell/csi-powerstore/v2/mocks" - "github.com/dell/csi-powerstore/v2/pkg/identifiers" - nfsmock "github.com/dell/csm-sharednfs/nfs/mocks" - "github.com/dell/gocsi" - csictx "github.com/dell/gocsi/context" - "github.com/fsnotify/fsnotify" - "github.com/sirupsen/logrus" - "github.com/spf13/viper" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "go.uber.org/mock/gomock" - "google.golang.org/grpc" -) - -func TestNew(t *testing.T) { - assert.NotNil(t, New()) -} - -func TestVolumeIDToArrayID(t *testing.T) { - t.Run("empty volume id", func(t *testing.T) { - resp := New().VolumeIDToArrayID("") - assert.Empty(t, resp) - }) - - t.Run("normal volume id", func(t *testing.T) { - resp := New().VolumeIDToArrayID("123-456") - assert.Equal(t, "123", resp) - }) -} - -func TestRegisterAdditionalServers(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockController := new(mocks.ControllerInterface) - mockController.On("RegisterAdditionalServers", mock.Anything).Return() - svc := New() - PutControllerService(mockController) - server := grpc.Server{} - svc.RegisterAdditionalServers(&server) -} - -func TestBeforeServe(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - t.Run("no node name", func(t *testing.T) { - mockNfs := nfsmock.NewMockService(ctrl) - mockNfs.EXPECT().BeforeServe(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(fmt.Errorf("no node name")) - PutNfsService(mockNfs) - - err := New().BeforeServe(context.Background(), &gocsi.StoragePlugin{}, nil) - assert.Nil(t, err) - }) - - t.Run("X_CSI_POWERSTORE_NODE_CHROOT_PATH", func(t *testing.T) { - os.Setenv("X_CSI_NODE_NAME", "test") - ctx := context.Background() - csictx.Setenv(ctx, gocsi.EnvVarMode, "node") - - err := New().BeforeServe(context.Background(), &gocsi.StoragePlugin{}, nil) - assert.Error(t, err, fmt.Errorf("X_CSI_POWERSTORE_NODE_CHROOT_PATH environment variable not set")) - }) - - t.Run("success", func(t *testing.T) { - os.Setenv("X_CSI_NODE_NAME", "test") - ctx := context.Background() - csictx.Setenv(ctx, gocsi.EnvVarMode, "node") - os.Setenv(identifiers.EnvNodeChrootPath, "test-path") - mockNfs := nfsmock.NewMockService(ctrl) - mockNfs.EXPECT().BeforeServe(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil) - PutNfsService(mockNfs) - assert.Nil(t, New().BeforeServe(ctx, &gocsi.StoragePlugin{}, nil)) - }) -} - -func TestProcessMapSecretChange(t *testing.T) { - err := New().ProcessMapSecretChange() - assert.Nil(t, err) -} - -func TestUpdateDriverConfigParams(t *testing.T) { - v := viper.New() - v.SetConfigType("yaml") - v.SetDefault("CSI_LOG_FORMAT", "text") - v.SetDefault("CSI_LOG_LEVEL", "debug") - - viperChan := make(chan bool) - v.WatchConfig() - v.OnConfigChange(func(_ fsnotify.Event) { - updateDriverConfigParams(v) - viperChan <- true - }) - - logFormat := strings.ToLower(v.GetString("CSI_LOG_FORMAT")) - assert.Equal(t, "text", logFormat) - - updateDriverConfigParams(v) - level := log.GetLevel() - - assert.Equal(t, logrus.DebugLevel, level) - - v.Set("CSI_LOG_FORMAT", "json") - v.Set("CSI_LOG_LEVEL", "info") - updateDriverConfigParams(v) - level = log.GetLevel() - assert.Equal(t, logrus.InfoLevel, level) - - v.Set("CSI_LOG_LEVEL", "notalevel") - updateDriverConfigParams(v) - level = log.GetLevel() - assert.Equal(t, logrus.DebugLevel, level) -} diff --git a/samples/secret/secret.yaml b/samples/secret/secret.yaml index 1340ffda..7bd6aa29 100644 --- a/samples/secret/secret.yaml +++ b/samples/secret/secret.yaml @@ -84,18 +84,129 @@ arrays: # Default value: "0777" # nfsAcls: "0777" - # Host based registration for powerstore metro - # To enable host based registration for powerstore metro, uncomment the following line - # metroTopology: This parameter will be used for host based registration + # metroTopology is used in conjunction with "labels" to provide host based + # registration for powerstore metro, where host nodes containing the provided + # label will be registered as co-located local hosts, and nodes with the label + # specified in the configuration for the metro replication target will be + # registered as co-located remote hosts. All other nodes are registered as + # co-located both hosts. + # + # *** Deprecated *** + # Use hostConnectivity instead. + # metroTopology is deprecated and remains only to support backward compatibility. + # # Allowed Values: Uniform # Default Value: Uniform + # + # Example: # metroTopology: Uniform - # This label will be used to match the node label to decide the type of host registration, only one label should be specified - # Allowed Values: : + # label is used in conjunction with metroTopology to determine if the host + # will be used to match the node label to decide the type of host registration, + # Only one label should be specified. + # + # *** Deprecated *** + # Use hostConnectivity instead. + # Use of "labels" with "metroTopology" is deprecated and remains to support + # backward compatibility. + # + # Optional: true + # Allowed Values: a single key-value pair # Default Value: None + # + # Example: # labels: - # : + # topology.kubernetes.io/zone: "zone-a" + + # hostConnectivity is used in conjunction with metro replication to configure + # multipath optimization for a node's host entry in this PowerStore system. + # + # UNIFORM METRO + # Uniform metro is configured under the `metro` field. + # For each desired optimization (`metro.colocatedLocal`, `metro.colocatedRemote`, + # `metro.colocatedBoth`), provide a set of label expressions to describe a node + # whose host should be registered with this PowerStore, with the given multipath + # optimization. + # + # If local, non-metro hosts are required alongside uniform metro hosts, under + # `hostConnectivity.local`, provide a set of label expressions to describe a node + # whose host should be registered with this PowerStore, without multipath optimization. + # + # NON-UNIFORM METRO + # Non-uniform host nodes should be registered as `local` and are configured + # under the `local` field. Non-uniform hosts in PowerStore do not require multipath + # optimization because the host nodes are only communicating with one side of the metro + # volume. + # To register a node as a non-uniform, metro host, under `hostConnectivity.local`, + # provide a set of label expressions to describe a node whose host should be + # registered with this PowerStore, without multipath optimization. + # + # matchExpressions follow the format outlined by Kubernetes when describing Node Affinity. + # + # Optional: true + # Default: none + # + # Example: Uniform metro with additional local, non-metro hosts + # hostConnectivity: + # local: + # nodeSelectorTerms: + # - matchExpressions: + # # This expression matches the remaining nodes that are not already described by + # # the metro topology described below. + # - key: "topology.kubernetes.io/zone" + # operator: "NotIn" + # values: + # - "zone-a" + # - "zone-b" + # - "zone-ab" + # metro: + # colocatedLocal: + # # A list of node selection terms. The terms are ORed. + # nodeSelectorTerms: + # # "matchExpressions" is a list of node selector requirements. + # # The requirements are ANDed. + # - matchExpressions: + # # "Key" is the label key that the selector applies to. + # - key: "topology.kubernetes.io/zone" + # # "Operator" represents a key's relationship to a set of values. + # # Allowed values: "In", "NotIn", "Exists", "DoesNotExist", "Gt", "Lt" + # operator: "In" + # # "Values" is an array of string values. + # # If the operator is In or NotIn, the values array must be non-empty. + # # If the operator is Exists or DoesNotExist, the values array must be empty. + # # If the operator is Gt or Lt, the values array must have a single element, + # # which will be interpreted as an integer. + # values: + # - "zone-a" + # colocatedRemote: + # nodeSelectorTerms: + # - matchExpressions: + # - key: "topology.kubernetes.io/zone" + # operator: "In" + # values: + # - "zone-b" + # colocatedBoth: + # nodeSelectorTerms: + # - matchExpressions: + # - key: "topology.kubernetes.io/zone" + # operator: "In" + # values: + # - "zone-ab" + # # OR: to successfully register a node, it must match AT LEAST ONE of the + # # "matchExpressions" under the "nodeSelectorTerms" + # - matchExpressions: + # - key: "topology.kubernetes.io/region" + # operator: "In" + # values: + # - "region-1" + # # AND: to successfully register a node, it must match ALL conditions + # # described by entries under "matchExpressions" + # - key: "topology.kubernetes.io/zone" + # operator: "NotIn" + # values: + # - "zone-a" + # - "zone-b" + # To add more PowerStore arrays, uncomment the following lines and provide the required values # - endpoint: "https://11.0.0.1/api/rest" @@ -104,3 +215,26 @@ arrays: # password: "password" # skipCertificateValidation: true # blockProtocol: "FC" +# hostConnectivity: +# metro: +# colocatedLocal: +# nodeSelectorTerms: +# - matchExpressions: +# - key: "topology.kubernetes.io/zone" +# operator: "In" +# values: +# - "zone-b" +# colocatedRemote: +# nodeSelectorTerms: +# - matchExpressions: +# - key: "topology.kubernetes.io/zone" +# operator: "In" +# values: +# - "zone-a" +# colocatedBoth: +# nodeSelectorTerms: +# - matchExpressions: +# - key: "topology.kubernetes.io/zone" +# operator: "In" +# values: +# - "zone-ab" diff --git a/samples/storageclass/powerstore-nfs-replication.yaml b/samples/storageclass/powerstore-nfs-replication.yaml new file mode 100644 index 00000000..a568bdf3 --- /dev/null +++ b/samples/storageclass/powerstore-nfs-replication.yaml @@ -0,0 +1,100 @@ +# +# +# Copyright © 2021-2024 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: "powerstore-nfs-replication" +provisioner: "csi-powerstore.dellemc.com" +reclaimPolicy: Delete +volumeBindingMode: Immediate +parameters: + # replicationPrefix paramater in values.yaml must be used as prefix for all replication parameters in storage class + # for e.g., all replication parameters have prefix: replication.storage.dell.com here + + # replication.storage.dell.com/isReplicationEnabled: + # Allowed values: + # true: replication is enabled + # false: replication is disabled + # Optional: true + # Default value: false + replication.storage.dell.com/isReplicationEnabled: "true" + + # replication.storage.dell.com/mode: replication mode + # Allowed values: + # "ASYNC" - Asynchronous mode + # "SYNC" - Synchronous mode + # "METRO" - Metro mode + # Optional: true + # Default value: "ASYNC" + replication.storage.dell.com/mode: "ASYNC" + + # replication.storage.dell.com/remoteStorageClassName: + # Allowed values: string + # Optional: true + # Default value: None + replication.storage.dell.com/remoteStorageClassName: "powerstore-replication" + + # replication.storage.dell.com/remoteClusterID: point to correct remote cluster id + # Allowed values: string + # Optional: true + # Default value: None + replication.storage.dell.com/remoteClusterID: "tgt-cluster-id" + + # replication.storage.dell.com/remoteSystem: point to correct remote PowerStore system + # Allowed values: string + # Optional: true + # Default value: None + replication.storage.dell.com/remoteSystem: "RT-0000" + + # replication.storage.dell.com/rpo: change to any other RPOs supported by PowerStore + # Allowed values: "Five_Minutes", "Fifteen_Minutes", "Thirty_Minutes", "One_Hour", "Six_Hours", "Twelve_Hours", "One_Day","Zero" + # Optional: true + # Default value: None + # For SYNC replication, this value must be set to Zero + replication.storage.dell.com/rpo: Five_Minutes + + # replication.storage.dell.com/ignoreNamespaces: set to 'true' if you want to ignore namespaces and use one volume group + # Allowed values: + # true: ignore namespaces and use one volume group + # false: create separate volume group per namespace + # Optional: true + # Default value: None + replication.storage.dell.com/ignoreNamespaces: "false" + + # replication.storage.dell.com/volumeGroupPrefix: volume group prefix + # Allowed values: string + # Optional: true + # Default value: None + replication.storage.dell.com/volumeGroupPrefix: "csi" + + # arrayID: id of array to be used for volumes + # Allowed values: arrayID corresponding to array's globalID specified in secret.yaml + # Optional: false + # Default value: None + arrayID: "Unique" + + # FsType: file system type for mounted volumes + # Allowed values: + # ext3: ext3 filesystem type + # ext4: ext4 filesystem type + # xfs: XFS filesystem type + # nfs: NFS filesystem + # Optional: true + # Default value: None if defaultFsType is not mentioned in values.yaml + # Else defaultFsType value mentioned in values.yaml + # will be used as default value + csi.storage.k8s.io/fstype: "nfs" diff --git a/tests/e2e/k8s/README.md b/tests/e2e/k8s/README.md deleted file mode 100644 index 7d2a8f08..00000000 --- a/tests/e2e/k8s/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# CSI PowerStore K8s E2E Tests - - -## Prerequisites -* A working Kubernetes cluster with csi-powerstore driver installed. -* A PowerStore storage system. - -## Test Setup -* Set the KUBECONFIG environment variable using `export KUBECONFIG=/path/to/.kube/config`, replacing the path with the path to your kubeconfig. -If $KUBECONFIG is unset, the test suite will attempt to locate the config under `$HOME/.kube/config`. -* Update `./e2e-values.yaml` with the necessary test values. - -## Running tests -Execute the run script to run the tests. -``` -./run.sh -``` - -## Updating Modules -The several modules imported from k8s.io appear to intentionally leave module versions set to v0.0.0. Without overwriting this version and specifying the module version, this test package will be unable to build its go.mod file. To fix this and specify the desired version, you must give the version using the 'replace' keyword in the go.mod file. - -When updating modules for this test package, make sure to update the list of replaced versions at the bottom of the go.mod file. diff --git a/tests/e2e/k8s/e2e-values.yaml b/tests/e2e/k8s/e2e-values.yaml deleted file mode 100644 index 9ab2978c..00000000 --- a/tests/e2e/k8s/e2e-values.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# -# -# Copyright © 2022 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -kubeconfigEnvVar: "KUBECONFIG" -busyBoxImageOnGcr: "gcr.io/google_containers/busybox:1.27" - -driverNamespace: "powerstore" -e2eCSIDriverName: "csi-powerstore.dellemc.com" -arrayID: "unique" - -diskSize: "3Gi" -execCommand: "while true ; do sleep 2 ; done" - -# config for externalAccess e2e test suite -externalAccess: - endPoint: "https://10.0.0.1/api/rest" # array's endpoint - userName: "user" - password: "password" - externalAccessIP: "10.0.0.1/25" # IP configured in values.yaml file while installing the driver - NASServer: "nas-server" - testStatefulset: true # if wanted to test ExternalAccess with Deployment as well as with Statefulset resource diff --git a/tests/e2e/k8s/externalAccess.go b/tests/e2e/k8s/externalAccess.go deleted file mode 100644 index 3ec682a3..00000000 --- a/tests/e2e/k8s/externalAccess.go +++ /dev/null @@ -1,303 +0,0 @@ -/* - * - * Copyright © 2022-2023 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package e2etest - -import ( - "context" - "crypto/rand" - "fmt" - "math/big" - "strconv" - "time" - - "github.com/dell/csi-powerstore/v2/pkg/identifiers" - "github.com/dell/gopowerstore" - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - - v1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/pod-security-admission/api" - - storagev1 "k8s.io/api/storage/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" - "k8s.io/kubernetes/test/e2e/framework/deployment" - fpv "k8s.io/kubernetes/test/e2e/framework/pv" - fss "k8s.io/kubernetes/test/e2e/framework/statefulset" -) - -/* - - Create NFS SC. - Create PVC using above SC. - Deployment having 10 replicas using above PVC. - Scale down to 1 pod. - Check if externalAccess is present in the NFS Export after scaling down to 1 pod - Scale down to 0 pods. - Cleanup the NS, i.e. SC, PVC, Pod. - - -*/ - -var ( - testParameters map[interface{}]interface{} - deploymentObject *v1.Deployment - mountPath = "/data" - extCredential ExternaAccess -) - -// ExternaAccess for storing ExternalAccess credentials -type ExternaAccess struct { - EndPoint string - UserName string - Password string - ExternalAccessIP string - NASServer string - TestStatefulset bool -} - -var _ = ginkgo.Describe("External Access Test", func() { - // Building a namespace api object, basename external-access - var ( - namespace string - client clientset.Interface - ) - - framework.TestContext.VerifyServiceAccount = false - f := framework.NewDefaultFramework("external-access") - // prevent annoying psp warning - - // f.SkipPrivilegedPSPBinding = true - defer ginkgo.GinkgoRecover() - framework.Logf("run e2e test default timeouts %#v ", f.Timeouts) - ginkgo.BeforeEach(func() { - namespace = getNamespaceToRunTests(f) - client = f.ClientSet - bootstrap() - }) - - ginkgo.AfterEach(func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - DeleteDeployment(client, deploymentObject, namespace) - if extCredential.TestStatefulset { - ginkgo.By(fmt.Sprintf("Deleting all statefulsets in namespace: %v", namespace)) - fss.DeleteAllStatefulSets(ctx, client, namespace) - } - }) - - // in case you want to log and exit framework.Fail("stop test") - - // Test for external Access feature check - ginkgo.It("[csi-externalAccess] Verify Host Access List for exteral access", func() { - curtime := time.Now().Unix() - nBig, err := rand.Int(rand.Reader, big.NewInt(27)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - randomValue := nBig.Int64() - val := strconv.FormatInt(int64(randomValue), 10) - curtimestring := strconv.FormatInt(curtime, 10) - scName := "exteral-access-sc-" + curtimestring + val - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - ginkgo.By("Creating NFS Storage Class & PVC") - - // get array credential from config file - extCredential = getExternalAccessCredential(testParameters["externalAccess"]) - - testParameters["allowRoot"] = "true" - testParameters["csi.storage.k8s.io/fstype"] = "nfs" - testParameters["nasName"] = extCredential.NASServer - - ds := fmt.Sprintf("%v", testParameters["diskSize"]) - storageclasspvc, pvclaim, err := createPVCAndStorageClass(client, - namespace, nil, testParameters, ds, nil, storagev1.VolumeBindingImmediate, true, "ReadWriteMany", scName) - - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, storageclasspvc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Expect PVC's claim status to be in Bound state") - err = fpv.WaitForPersistentVolumeClaimPhase(ctx, corev1.ClaimBound, client, - pvclaim.Namespace, pvclaim.Name, framework.Poll, 2*time.Minute) - - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Failed to find the volume in bound state with err: %v", err)) - - podLabels := map[string]string{ - "app": "dell-wip", - } - // Deployment is a resource to deploy a stateless application, if using a PVC, all replicas will be using the same Volume - ginkgo.By("Creating Deployment having 10 replicas") - deploymentObject, err = deployment.CreateDeployment(ctx, client, int32(10), podLabels, nil, namespace, []*corev1.PersistentVolumeClaim{pvclaim}, api.LevelPrivileged, fmt.Sprintf("%v", testParameters["execCommand"])) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Failed to create deployment resource with err: %v", err)) - - ginkgo.By("Deployment got created") - // now get the pvc, more intrested in volumeId - v := getPvFromClaim(client, namespace, pvclaim.Name) - framework.Logf("Volume Name that got attached to all pods: %s", v.GetName()) - - ginkgo.By("Scaling down to 1 replica") - ScaleDownDeployment(client, deploymentObject, namespace, 1) - - // fetch the NFS export object from array to avoid conflict in the response - // Get host access list from array for above volume - clientOptions := gopowerstore.NewClientOptions() - clientOptions.SetInsecure(true) - clientForArray, err := gopowerstore.NewClientWithArgs( - extCredential.EndPoint, extCredential.UserName, extCredential.Password, clientOptions) - - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Failed to connect with PowerStore Array, err: %v", err)) - checkExternalAccessPresence(ctx, clientForArray, extCredential.ExternalAccessIP, v.GetName(), true) - - // now in NFS Export only externalIP will be present and other node's IP will be deleted - ScaleDownDeployment(client, deploymentObject, namespace, 0) - checkExternalAccessPresence(ctx, clientForArray, extCredential.ExternalAccessIP, v.GetName(), true) - - err = fpv.DeletePersistentVolumeClaim(ctx, client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Unable to delete PVC with err: %v", err)) - - if extCredential.TestStatefulset { - // statefulset logic - scName2 := scName + "stateful-set" - sc, err := createStorageClass(client, testParameters, nil, "", - storagev1.VolumeBindingImmediate, - true, scName2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - statefulset := GetStatefulSetFromManifest(namespace) - ginkgo.By("Creating statefulset") - - statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1].Spec.AccessModes[0] = corev1.ReadWriteMany - - statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Annotations["volume.beta.kubernetes.io/storage-class"] = scName2 - - CreateStatefulSet(namespace, statefulset, client) - - defer func() { - ginkgo.By(fmt.Sprintf("Deleting all statefulsets in namespace: %v", namespace)) - fss.DeleteAllStatefulSets(ctx, client, namespace) - }() - replicas := *(statefulset.Spec.Replicas) - // Waiting for pods status to be Ready - fss.WaitForStatusReadyReplicas(ctx, client, statefulset, replicas) - - gomega.Expect(fss.CheckMount(ctx, client, statefulset, mountPath)).NotTo(gomega.HaveOccurred()) - - ssPodsBeforeScaleDown := fss.GetPodList(ctx, client, statefulset) - gomega.Expect(ssPodsBeforeScaleDown.Items).NotTo(gomega.BeEmpty(), - fmt.Sprintf("Unable to get list of Pods from the Statefulset: %v", statefulset.Name)) - gomega.Expect(len(ssPodsBeforeScaleDown.Items) == int(replicas)).To(gomega.BeTrue(), - "Number of Pods in the statefulset should match with number of replicas") - - // Get the list of Volumes attached to Pods before scale down - for _, sspod := range ssPodsBeforeScaleDown.Items { - _, err := client.CoreV1().Pods(namespace).Get(ctx, sspod.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, volumespec := range sspod.Spec.Volumes { - if volumespec.PersistentVolumeClaim != nil { - pv := getPvFromClaim(client, statefulset.Namespace, volumespec.PersistentVolumeClaim.ClaimName) - gomega.Expect(pv).NotTo(gomega.BeNil()) - } - } - } - - // 1 - replicas = 1 - ginkgo.By(fmt.Sprintf("Scaling down statefulsets to number of Replica: %v", replicas)) - _, scaledownErr := fss.Scale(ctx, client, statefulset, replicas) - gomega.Expect(scaledownErr).NotTo(gomega.HaveOccurred()) - fss.WaitForStatusReplicas(ctx, client, statefulset, replicas) - ssPodsAfterScaleDown := fss.GetPodList(ctx, client, statefulset) - gomega.Expect(ssPodsAfterScaleDown.Items).NotTo(gomega.BeEmpty(), - fmt.Sprintf("Unable to get list of Pods from the Statefulset: %v", statefulset.Name)) - gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(replicas)).To(gomega.BeTrue(), - "Number of Pods in the statefulset should match with number of replicas") - - var pv *corev1.PersistentVolume - // Get the list of Volumes attached to Pods after scale down - // for us one iteration is also okay to have since PV Id/Name will be same. - for _, sspod := range ssPodsAfterScaleDown.Items { - _, err := client.CoreV1().Pods(namespace).Get(ctx, sspod.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, volumespec := range sspod.Spec.Volumes { - if volumespec.PersistentVolumeClaim != nil { - pv = getPvFromClaim(client, statefulset.Namespace, volumespec.PersistentVolumeClaim.ClaimName) - gomega.Expect(pv).NotTo(gomega.BeNil()) - } - } - } - - checkExternalAccessPresence(ctx, clientForArray, extCredential.ExternalAccessIP, pv.GetName(), true) - - // deleting all pods - replicas = 0 - ginkgo.By(fmt.Sprintf("Scaling down statefulsets to number of Replica: %v", replicas)) - _, scaledownErr = fss.Scale(ctx, client, statefulset, replicas) - gomega.Expect(scaledownErr).NotTo(gomega.HaveOccurred()) - fss.WaitForStatusReplicas(ctx, client, statefulset, replicas) - ssPodsAfterScaleDown = fss.GetPodList(ctx, client, statefulset) - gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(replicas)).To(gomega.BeTrue(), - "Number of Pods in the statefulset should match with number of replicas") - - err = fpv.DeletePersistentVolumeClaim(ctx, client, pv.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Unable to delete PVC with err: %v", err)) - } - }) -}) - -func getExternalAccessCredential(credential interface{}) (ext ExternaAccess) { - myMap := credential.(map[interface{}]interface{}) - ext.EndPoint = fmt.Sprintf("%v", myMap["endPoint"]) - ext.UserName = fmt.Sprintf("%v", myMap["userName"]) - ext.Password = fmt.Sprintf("%v", myMap["password"]) - ext.NASServer = fmt.Sprintf("%v", myMap["NASServer"]) - ext.ExternalAccessIP = fmt.Sprintf("%v", myMap["externalAccessIP"]) - ext.TestStatefulset, _ = strconv.ParseBool(fmt.Sprintf("%v", myMap["testStatefulset"])) - return ext -} - -func checkExternalAccessPresence(ctx context.Context, clientForArray gopowerstore.Client, externalAccessIP string, vol string, shouldBePresent bool) { - nfsExport, err := clientForArray.GetNFSExportByName(ctx, vol) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Failed to GET NFS export details from Array, err: %v", err)) - present := identifiers.ExternalAccessAlreadyAdded(nfsExport, externalAccessIP) - if shouldBePresent && !present { - gomega.Expect(present).NotTo(gomega.BeFalse(), - "External access should be present on host access list on array") - } - if !shouldBePresent && present { - gomega.Expect(present).NotTo(gomega.BeTrue(), - "External access should not be present on host access list on array") - } -} diff --git a/tests/e2e/k8s/go.mod b/tests/e2e/k8s/go.mod deleted file mode 100644 index debee3a2..00000000 --- a/tests/e2e/k8s/go.mod +++ /dev/null @@ -1,212 +0,0 @@ -module github.com/dell/csi-powerstore/v2/tests/e2e - -go 1.25 - -require ( - github.com/dell/csi-powerstore/v2 v2.11.0 - github.com/dell/gopowerstore v1.19.1-0.20250828071553-729e69f22fc3 - github.com/onsi/ginkgo/v2 v2.23.4 - github.com/onsi/gomega v1.38.0 - gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.34.0 - k8s.io/apimachinery v0.34.0 - k8s.io/client-go v0.34.0 - k8s.io/kubernetes v1.31.12 - k8s.io/pod-security-admission v0.33.0 -) - -require ( - cel.dev/expr v0.24.0 // indirect - github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect - github.com/Microsoft/go-winio v0.6.0 // indirect - github.com/Microsoft/hcsshim v0.8.26 // indirect - github.com/NYTimes/gziphandler v1.1.1 // indirect - github.com/antlr4-go/antlr/v4 v4.13.0 // indirect - github.com/apparentlymart/go-cidr v1.1.0 // indirect - github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e // indirect - github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/checkpoint-restore/go-criu/v5 v5.3.0 // indirect - github.com/cilium/ebpf v0.9.1 // indirect - github.com/container-storage-interface/spec v1.10.0 // indirect - github.com/containerd/cgroups v1.1.0 // indirect - github.com/containerd/console v1.0.3 // indirect - github.com/containerd/ttrpc v1.2.2 // indirect - github.com/coreos/go-semver v0.3.1 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect - github.com/cyphar/filepath-securejoin v0.2.4 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/dell/gobrick v1.14.1 // indirect - github.com/dell/gocsi v1.14.1-0.20250828071850-fe2891b95da5 // indirect - github.com/dell/gofsutil v1.19.1-0.20250828071543-d6f6a5a812e3 // indirect - github.com/dell/goiscsi v1.12.1-0.20250828071455-fd1c391bd920 // indirect - github.com/dell/gonvme v1.11.1 // indirect - github.com/distribution/reference v0.5.0 // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/emicklei/go-restful/v3 v3.12.2 // indirect - github.com/euank/go-kmsg-parser v2.0.0+incompatible // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.9.0 // indirect - github.com/fxamacker/cbor/v2 v2.9.0 // indirect - github.com/go-logr/logr v1.4.3 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/errors v0.22.0 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.21.0 // indirect - github.com/go-openapi/strfmt v0.23.0 // indirect - github.com/go-openapi/swag v0.23.0 // indirect - github.com/go-task/slim-sprig/v3 v3.0.0 // indirect - github.com/godbus/dbus/v5 v5.1.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/mock v1.6.0 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/cadvisor v0.49.0 // indirect - github.com/google/cel-go v0.23.2 // indirect - github.com/google/gnostic-models v0.7.0 // indirect - github.com/google/go-cmp v0.7.0 // indirect - github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/gorilla/websocket v1.5.0 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect - github.com/imdario/mergo v0.3.16 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/karrick/godirwalk v1.17.0 // indirect - github.com/kylelemons/godebug v1.1.0 // indirect - github.com/mailru/easyjson v0.9.0 // indirect - github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/moby/spdystream v0.4.0 // indirect - github.com/moby/sys/mountinfo v0.7.1 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect - github.com/mrunalp/fileutils v0.5.1 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect - github.com/oklog/ulid v1.3.1 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/runc v1.1.14 // indirect - github.com/opencontainers/runtime-spec v1.0.3-0.20220909204839-494a5a6aca78 // indirect - github.com/opencontainers/selinux v1.11.0 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.22.0 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.62.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - github.com/seccomp/libseccomp-golang v0.10.0 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect - github.com/spf13/cobra v1.9.1 // indirect - github.com/spf13/pflag v1.0.6 // indirect - github.com/stoewer/go-strcase v1.3.0 // indirect - github.com/stretchr/objx v0.5.2 // indirect - github.com/stretchr/testify v1.11.0 // indirect - github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect - github.com/vishvananda/netlink v1.1.0 // indirect - github.com/vishvananda/netns v0.0.4 // indirect - github.com/x448/float16 v0.8.4 // indirect - go.etcd.io/etcd/api/v3 v3.6.1 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.6.1 // indirect - go.etcd.io/etcd/client/v3 v3.6.1 // indirect - go.mongodb.org/mongo-driver v1.17.2 // indirect - go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect - go.opentelemetry.io/otel v1.37.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 // indirect - go.opentelemetry.io/otel/metric v1.37.0 // indirect - go.opentelemetry.io/otel/sdk v1.37.0 // indirect - go.opentelemetry.io/otel/trace v1.37.0 // indirect - go.opentelemetry.io/proto/otlp v1.5.0 // indirect - go.uber.org/automaxprocs v1.6.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect - go.yaml.in/yaml/v2 v2.4.2 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.41.0 // indirect - golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect - golang.org/x/mod v0.26.0 // indirect - golang.org/x/net v0.43.0 // indirect - golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.35.0 // indirect - golang.org/x/term v0.34.0 // indirect - golang.org/x/text v0.28.0 // indirect - golang.org/x/time v0.9.0 // indirect - golang.org/x/tools v0.35.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect - google.golang.org/grpc v1.75.0 // indirect - google.golang.org/protobuf v1.36.6 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.0.0 // indirect - k8s.io/apiserver v0.33.0 // indirect - k8s.io/cloud-provider v0.0.0 // indirect - k8s.io/component-base v0.33.0 // indirect - k8s.io/component-helpers v0.31.1 // indirect - k8s.io/controller-manager v0.31.1 // indirect - k8s.io/cri-api v0.31.1 // indirect - k8s.io/cri-client v0.0.0 // indirect - k8s.io/csi-translation-lib v0.0.0 // indirect - k8s.io/dynamic-resource-allocation v0.31.1 // indirect - k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kms v0.33.0 // indirect - k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect - k8s.io/kube-scheduler v0.0.0 // indirect - k8s.io/kubectl v0.0.0 // indirect - k8s.io/kubelet v0.31.1 // indirect - k8s.io/mount-utils v0.0.0 // indirect - k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect - sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect - sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect -) - -replace ( - github.com/dell/csi-powerstore/v2 => ../../../ - k8s.io/api => k8s.io/api v0.31.1 - k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.31.1 - k8s.io/apimachinery => k8s.io/apimachinery v0.31.1 - k8s.io/apiserver => k8s.io/apiserver v0.31.1 - k8s.io/cli-runtime => k8s.io/cli-runtime v0.31.1 - k8s.io/client-go => k8s.io/client-go v0.31.1 - k8s.io/cloud-provider => k8s.io/cloud-provider v0.31.1 - k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.31.1 - k8s.io/code-generator => k8s.io/code-generator v0.31.1 - k8s.io/component-base => k8s.io/component-base v0.31.1 - k8s.io/component-helpers => k8s.io/component-helpers v0.31.1 - k8s.io/controller-manager => k8s.io/controller-manager v0.31.1 - k8s.io/cri-api => k8s.io/cri-api v0.31.1 - k8s.io/cri-client => k8s.io/cri-client v0.31.1 - k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.31.1 - k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.31.1 - k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.31.1 - k8s.io/kube-proxy => k8s.io/kube-proxy v0.31.1 - k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.31.1 - k8s.io/kubectl => k8s.io/kubectl v0.31.1 - k8s.io/kubelet => k8s.io/kubelet v0.31.1 - k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.31.1 - k8s.io/metrics => k8s.io/metrics v0.31.1 - k8s.io/mount-utils => k8s.io/mount-utils v0.31.1 - k8s.io/node-api => k8s.io/node-api v0.31.1 - k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.31.1 - k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.31.1 - k8s.io/sample-controller => k8s.io/sample-controller v0.31.1 - sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.11.1 -) diff --git a/tests/e2e/k8s/go.sum b/tests/e2e/k8s/go.sum deleted file mode 100644 index 83dfbeb3..00000000 --- a/tests/e2e/k8s/go.sum +++ /dev/null @@ -1,700 +0,0 @@ -bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= -cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= -cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab h1:UKkYhof1njT1/xq4SEg5z+VpTgjmNeHwPGRQl7takDI= -github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA= -github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= -github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= -github.com/Microsoft/hcsshim v0.8.26 h1:770C4dtDITZUaMQ9d6lVPdM8Lq4S0E0Tthy6T91mDMo= -github.com/Microsoft/hcsshim v0.8.26/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= -github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= -github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= -github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU= -github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= -github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/checkpoint-restore/go-criu/v5 v5.3.0 h1:wpFFOoomK3389ue2lAb0Boag6XPht5QYpipxmSNL4d8= -github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= -github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.9.1 h1:64sn2K3UKw8NbP/blsixRpF3nXuyhz/VjRlRzvlBRu4= -github.com/cilium/ebpf v0.9.1/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/container-storage-interface/spec v1.10.0 h1:YkzWPV39x+ZMTa6Ax2czJLLwpryrQ+dPesB34mrRMXA= -github.com/container-storage-interface/spec v1.10.0/go.mod h1:DtUvaQszPml1YJfIK7c00mlv6/g4wNMLanLgiUbKFRI= -github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= -github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= -github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= -github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= -github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= -github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= -github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= -github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= -github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= -github.com/containerd/ttrpc v1.2.2 h1:9vqZr0pxwOF5koz6N0N3kJ0zDHokrcPxIR/ZR2YFtOs= -github.com/containerd/ttrpc v1.2.2/go.mod h1:sIT6l32Ph/H9cvnJsfXM5drIVzTr5A2flTf1G5tYZak= -github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY= -github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= -github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= -github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dell/dell-csi-extensions/common v1.8.1-0.20250825155821-05e7f81ae500 h1:JMnNbdD0Fc/umW3vDSfQYsw8OKYFlmqMTVfHiA748LM= -github.com/dell/dell-csi-extensions/common v1.8.1-0.20250825155821-05e7f81ae500/go.mod h1:BS94dd2/oGCkWqZW4XeXHf6J/DssSy9vYUd4sb2VDB0= -github.com/dell/gobrick v1.14.1 h1:EzJ0GRacyA6kgU0l54WqYctWFtBrkpgm7WWutzUNmUQ= -github.com/dell/gobrick v1.14.1/go.mod h1:P1VwniwVCTIn5ZUvWZOpD0Xw/dn0LPzW4t2y54iAhEU= -github.com/dell/gocsi v1.14.1-0.20250828071850-fe2891b95da5 h1:MTI9h3sFmd0VBGnAMzkllUf/ftvBSfCOkG8PrpsCAIY= -github.com/dell/gocsi v1.14.1-0.20250828071850-fe2891b95da5/go.mod h1:GQO6RkslOBdpp4q52A+DAYywjaRpm0JTwnkBm3WDFt8= -github.com/dell/gofsutil v1.19.1-0.20250828071543-d6f6a5a812e3 h1:f5QEavSh1QRmcmEKlwqOhBNlhDhXhudcATyJFdBAw20= -github.com/dell/gofsutil v1.19.1-0.20250828071543-d6f6a5a812e3/go.mod h1:nyEdtCcz19HCt4njR18JoqA9FJBA/QpbC6Z9QnwOAfM= -github.com/dell/goiscsi v1.12.1-0.20250828071455-fd1c391bd920 h1:YImEtK58XYtc/JmAm0bQ3nHB+Clj4STgy4JcVKiHJGA= -github.com/dell/goiscsi v1.12.1-0.20250828071455-fd1c391bd920/go.mod h1:nMLnEG6QBpouHxFRMxpmJA6X+Nrr3KeN9mr5n2gV+5M= -github.com/dell/gonvme v1.11.1 h1:D6iItc0xYMPm6A4m+2ONnFy/piYy+coQmwgyjNEMVSE= -github.com/dell/gonvme v1.11.1/go.mod h1:3bgVCRevHuVWR4UOV2uv5UksaJP5SiCzrgQZHqihaYA= -github.com/dell/gopowerstore v1.19.1-0.20250828071553-729e69f22fc3 h1:KGU0HvPRCem74V8yz9V8Vv58ek4JjfsPUZm46XzbQ/A= -github.com/dell/gopowerstore v1.19.1-0.20250828071553-729e69f22fc3/go.mod h1:ZEF/MMMvPk/JdPeKIUGZFXd+AgQxVqCw/NePpV3QcHc= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= -github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v20.10.27+incompatible h1:Id/ZooynV4ZlD6xX20RCd3SR0Ikn7r4QZDa2ECK2TgA= -github.com/docker/docker v20.10.27+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= -github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/euank/go-kmsg-parser v2.0.0+incompatible h1:cHD53+PLQuuQyLZeriD1V/esuG4MuU0Pjs5y6iknohY= -github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= -github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= -github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= -github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= -github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= -github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= -github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= -github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c= -github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= -github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= -github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= -github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= -github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/cadvisor v0.49.0 h1:1PYeiORXmcFYi609M4Qvq5IzcvcVaWgYxDt78uH8jYA= -github.com/google/cadvisor v0.49.0/go.mod h1:s6Fqwb2KiWG6leCegVhw4KW40tf9f7m+SF1aXiE8Wsk= -github.com/google/cel-go v0.23.2 h1:UdEe3CvQh3Nv+E/j9r1Y//WO0K0cSyD7/y0bzyLIMI4= -github.com/google/cel-go v0.23.2/go.mod h1:52Pb6QsDbC5kvgxvZhiL9QX1oZEkcUF/ZqaPx1J5Wwo= -github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= -github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= -github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= -github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= -github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= -github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= -github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= -github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jarcoal/httpmock v1.4.0 h1:BvhqnH0JAYbNudL2GMJKgOHe2CtKlzJ/5rWKyp+hc2k= -github.com/jarcoal/httpmock v1.4.0/go.mod h1:ftW1xULwo+j0R0JJkJIIi7UKigZUXCLLanykgjwBXL0= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= -github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= -github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= -github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= -github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= -github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8= -github.com/moby/spdystream v0.4.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= -github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= -github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= -github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mrunalp/fileutils v0.5.1 h1:F+S7ZlNKnrwHfSwdlgNSkKo67ReVf8o9fel6C3dkm/Q= -github.com/mrunalp/fileutils v0.5.1/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= -github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= -github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= -github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= -github.com/onsi/gomega v1.38.0 h1:c/WX+w8SLAinvuKKQFh77WEucCnPk4j2OTUr7lt7BeY= -github.com/onsi/gomega v1.38.0/go.mod h1:OcXcwId0b9QsE7Y49u+BTrL4IdKOBOKnD6VQNTJEB6o= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= -github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v1.1.14 h1:rgSuzbmgz5DUJjeSnw337TxDbRuqjs6iqQck/2weR6w= -github.com/opencontainers/runc v1.1.14/go.mod h1:E4C2z+7BxR7GHXp0hAY53mek+x49X1LjPNeMTfRGvOA= -github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20220909204839-494a5a6aca78 h1:R5M2qXZiK/mWPMT4VldCOiSL9HIAMuxQZWdG0CSM5+4= -github.com/opencontainers/runtime-spec v1.0.3-0.20220909204839-494a5a6aca78/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= -github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= -github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= -github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= -github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= -github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/seccomp/libseccomp-golang v0.10.0 h1:aA4bp+/Zzi0BnWZ2F1wgNBs5gTpm+na2rWM6M9YjLpY= -github.com/seccomp/libseccomp-golang v0.10.0/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= -github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= -github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.11.0 h1:ib4sjIrwZKxE5u/Japgo/7SJV3PvgjGiRNAvTVGqQl8= -github.com/stretchr/testify v1.11.0/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= -github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= -github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= -github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= -github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= -github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= -github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= -github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= -github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= -github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= -go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.etcd.io/etcd/api/v3 v3.6.1 h1:yJ9WlDih9HT457QPuHt/TH/XtsdN2tubyxyQHSHPsEo= -go.etcd.io/etcd/api/v3 v3.6.1/go.mod h1:lnfuqoGsXMlZdTJlact3IB56o3bWp1DIlXPIGKRArto= -go.etcd.io/etcd/client/pkg/v3 v3.6.1 h1:CxDVv8ggphmamrXM4Of8aCC8QHzDM4tGcVr9p2BSoGk= -go.etcd.io/etcd/client/pkg/v3 v3.6.1/go.mod h1:aTkCp+6ixcVTZmrJGa7/Mc5nMNs59PEgBbq+HCmWyMc= -go.etcd.io/etcd/client/v3 v3.6.1 h1:KelkcizJGsskUXlsxjVrSmINvMMga0VWwFF0tSPGEP0= -go.etcd.io/etcd/client/v3 v3.6.1/go.mod h1:fCbPUdjWNLfx1A6ATo9syUmFVxqHH9bCnPLBZmnLmMY= -go.etcd.io/etcd/pkg/v3 v3.6.1 h1:Qpshk3/SLra217k7FxcFGaH2niFAxFf1Dug57f0IUiw= -go.etcd.io/etcd/pkg/v3 v3.6.1/go.mod h1:nS0ahQoZZ9qXjQAtYGDt80IEHKl9YOF7mv6J0lQmBoQ= -go.etcd.io/etcd/server/v3 v3.6.1 h1:Y/mh94EeImzXyTBIMVgR0v5H+ANtRFDY4g1s5sxOZGE= -go.etcd.io/etcd/server/v3 v3.6.1/go.mod h1:nCqJGTP9c2WlZluJB59j3bqxZEI/GYBfQxno0MguVjE= -go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ= -go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo= -go.mongodb.org/mongo-driver v1.17.2 h1:gvZyk8352qSfzyZ2UMWcpDpMSGEr1eqE4T793SqyhzM= -go.mongodb.org/mongo-driver v1.17.2/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0 h1:KemlMZlVwBSEGaO91WKgp41BBFsnWqqj9sKRwmOqC40= -go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0/go.mod h1:uq8DrRaen3suIWTpdR/JNHCGpurSvMv9D5Nr5CU5TXc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 h1:rgMkmiGfix9vFJDcDi1PK8WEQP4FLQwLDfhp5ZLpFeE= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0/go.mod h1:ijPqXp5P6IRRByFVVg9DY8P5HkxkHE5ARIa+86aXPf4= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= -go.opentelemetry.io/contrib/propagators/b3 v1.19.0 h1:ulz44cpm6V5oAeg5Aw9HyqGFMS6XM7untlMEhD7YzzA= -go.opentelemetry.io/contrib/propagators/b3 v1.19.0/go.mod h1:OzCmE2IVS+asTI+odXQstRGVfXQ4bXv9nMBRK0nNyqQ= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= -go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= -go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= -go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= -go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= -go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= -go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= -go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= -go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= -go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= -go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= -golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= -golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= -golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= -golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= -golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= -golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 h1:FiusG7LWj+4byqhbvmB+Q93B/mOxJLN2DTozDuZm4EU= -google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:kXqgZtrWaf6qS3jZOCnCH7WYfrvFjkC51bM8fz3RsCA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= -google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= -gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.31.1 h1:Xe1hX/fPW3PXYYv8BlozYqw63ytA92snr96zMW9gWTU= -k8s.io/api v0.31.1/go.mod h1:sbN1g6eY6XVLeqNsZGLnI5FwVseTrZX7Fv3O26rhAaI= -k8s.io/apiextensions-apiserver v0.31.1 h1:L+hwULvXx+nvTYX/MKM3kKMZyei+UiSXQWciX/N6E40= -k8s.io/apiextensions-apiserver v0.31.1/go.mod h1:tWMPR3sgW+jsl2xm9v7lAyRF1rYEK71i9G5dRtkknoQ= -k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U= -k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= -k8s.io/apiserver v0.31.1 h1:Sars5ejQDCRBY5f7R3QFHdqN3s61nhkpaX8/k1iEw1c= -k8s.io/apiserver v0.31.1/go.mod h1:lzDhpeToamVZJmmFlaLwdYZwd7zB+WYRYIboqA1kGxM= -k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0= -k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg= -k8s.io/cloud-provider v0.31.1 h1:40b6AgDizwm5eWratZbqubTHMob25VWr6NX2Ei5TwZA= -k8s.io/cloud-provider v0.31.1/go.mod h1:xAdkE7fdZdu9rKLuOZUMBfagu7bM+bas3iPux/2nLGg= -k8s.io/component-base v0.31.1 h1:UpOepcrX3rQ3ab5NB6g5iP0tvsgJWzxTyAo20sgYSy8= -k8s.io/component-base v0.31.1/go.mod h1:WGeaw7t/kTsqpVTaCoVEtillbqAhF2/JgvO0LDOMa0w= -k8s.io/component-helpers v0.31.1 h1:5hZUf3747atdgtR3gPntrG35rC2CkK7rYq2KUraz6Os= -k8s.io/component-helpers v0.31.1/go.mod h1:ye0Gi8KzFNTfpIuzvVDtxJQMP/0Owkukf1vGf22Hl6U= -k8s.io/controller-manager v0.31.1 h1:bwiy8y//EG5lJL2mdbOvZWrOgw2EXXIvwp95VYgoIis= -k8s.io/controller-manager v0.31.1/go.mod h1:O440MSE6EI1AEVhB2Fc8FYqv6r8BHrSXjm5aj3886No= -k8s.io/cri-api v0.31.1 h1:x0aI8yTI7Ho4c8tpuig8NwI/MRe+VhjiYyyebC2xphQ= -k8s.io/cri-api v0.31.1/go.mod h1:Po3TMAYH/+KrZabi7QiwQI4a692oZcUOUThd/rqwxrI= -k8s.io/cri-client v0.31.1 h1:w5D7BAhiaSVVDZqHs7YUZPpuUCybx8tCxfdBuDBw7zo= -k8s.io/cri-client v0.31.1/go.mod h1:voVfZexZQwvlf/JD8w30sGN0k22LRcHRfCj7+m4kAXE= -k8s.io/csi-translation-lib v0.31.1 h1:ps9kya8+ih0CVL59JO2B4AYH8U/e3WLQxl9sx19NjjM= -k8s.io/csi-translation-lib v0.31.1/go.mod h1:VeYSucPZJbAt6RT25AzfG7WjyxCcmqxtr4V/CaDdNZc= -k8s.io/dynamic-resource-allocation v0.31.1 h1:AiOVtBdeBmKMbwAVnHmL/v+m9gY2z734x0LKJb4WOMg= -k8s.io/dynamic-resource-allocation v0.31.1/go.mod h1:I1j9Vk9/rbzAckolbNZg8WasttD5yYnsZeDX2dpISKQ= -k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= -k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kms v0.33.0 h1:fhQSW/vyaWDhMp0vDuO/sLg2RlGZf4F77beSXcB4/eE= -k8s.io/kms v0.33.0/go.mod h1:C1I8mjFFBNzfUZXYt9FZVJ8MJl7ynFbGgZFbBzkBJ3E= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= -k8s.io/kube-scheduler v0.31.1 h1:hbTiOUqEgPuXa85/J2ZYzIK7aYZruuOaQAirv5TQXjQ= -k8s.io/kube-scheduler v0.31.1/go.mod h1:pJKhtHJthZbxXpF+Mecb0wPXecYxsiMJbhuNi0xUsrE= -k8s.io/kubectl v0.31.1 h1:ih4JQJHxsEggFqDJEHSOdJ69ZxZftgeZvYo7M/cpp24= -k8s.io/kubectl v0.31.1/go.mod h1:aNuQoR43W6MLAtXQ/Bu4GDmoHlbhHKuyD49lmTC8eJM= -k8s.io/kubelet v0.31.1 h1:aAxwVxGzbbMKKk/FnSjvkN52K3LdHhjhzmYcyGBuE0c= -k8s.io/kubelet v0.31.1/go.mod h1:8ZbexYHqUO946gXEfFmnMZiK2UKRGhk7LlGvJ71p2Ig= -k8s.io/kubernetes v1.31.12 h1:dPgK1slI7p/D3I2J1NA6UfBeMMHcjB91rHdXMpx8fkU= -k8s.io/kubernetes v1.31.12/go.mod h1:9xmT2buyTYj8TRKwRae7FcuY8k5+xlxv7VivvO0KKfs= -k8s.io/mount-utils v0.31.1 h1:f8UrH9kRynljmdNGM6BaCvFUON5ZPKDgE+ltmYqI4wA= -k8s.io/mount-utils v0.31.1/go.mod h1:HV/VYBUGqYUj4vt82YltzpWvgv8FPg0G9ItyInT3NPU= -k8s.io/pod-security-admission v0.33.0 h1:di/iicB5plCq+iQeqgf2s1N5DOSzTDiOOv5OiAbuYWE= -k8s.io/pod-security-admission v0.33.0/go.mod h1:McuUMtSclLNxQdCkDTTWqKR79jnpHT/022GuanVU/Wg= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= -sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= -sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= -sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= -sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= -sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/tests/e2e/k8s/run.sh b/tests/e2e/k8s/run.sh deleted file mode 100755 index ff39c509..00000000 --- a/tests/e2e/k8s/run.sh +++ /dev/null @@ -1,23 +0,0 @@ - -# -# -# Copyright © 2022 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -# supress ginkgo 2.0 upgrade hints -export ACK_GINKGO_DEPRECATIONS=1.16.5 - -# run all tests -go test -timeout=100m -v ./ -ginkgo.v=1 - diff --git a/tests/e2e/k8s/suite_test.go b/tests/e2e/k8s/suite_test.go deleted file mode 100644 index ce7eeee5..00000000 --- a/tests/e2e/k8s/suite_test.go +++ /dev/null @@ -1,76 +0,0 @@ -/* - * - * Copyright © 2022-2023 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package e2etest - -import ( - "flag" - "fmt" - "os" - "path/filepath" - "testing" - - ginkgo "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/reporters" - gomega "github.com/onsi/gomega" - "k8s.io/kubernetes/test/e2e/framework" - config "k8s.io/kubernetes/test/e2e/framework/config" -) - -func init() { - var yamlError error - - testParameters, yamlError = readYaml("e2e-values.yaml") - if yamlError != nil { - framework.Failf("Unable to read yaml e2e-values.yaml: %s", yamlError.Error()) - } - - // k8s.io/kubernetes/tests/e2e/framework requires env KUBECONFIG to be set - // it does not fall back to defaults - if os.Getenv(fmt.Sprintf("%v", testParameters["kubeconfigEnvVar"])) == "" { - kubeconfig := filepath.Join(os.Getenv("HOME"), ".kube", "config") - os.Setenv(fmt.Sprintf("%v", testParameters["kubeconfigEnvVar"]), kubeconfig) - } - - framework.TestContext.Provider = "local" - - t := framework.TestContextType{} - - framework.AfterReadingAllFlags(&t) -} - -func TestE2E(t *testing.T) { - handleFlags() - gomega.RegisterFailHandler(ginkgo.Fail) - - // pass/fail/skip results summarized to this file - junitReporter := reporters.NewJUnitReporter("junit.xml") - - // dont dump huge logs of node / pods on error - framework.TestContext.DumpLogsOnFailure = false - - // framework.TestContext.DeleteNamespace = false - - // runs all ginkgo tests in go files - ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "CSI Driver End-to-End Tests", []ginkgo.Reporter{junitReporter}) -} - -func handleFlags() { - config.CopyFlags(config.Flags, flag.CommandLine) - framework.RegisterCommonFlags(flag.CommandLine) - framework.RegisterClusterFlags(flag.CommandLine) - flag.Parse() -} diff --git a/tests/e2e/k8s/testing-manifests/statefulset/statefulset.yaml b/tests/e2e/k8s/testing-manifests/statefulset/statefulset.yaml deleted file mode 100644 index 1fa03af4..00000000 --- a/tests/e2e/k8s/testing-manifests/statefulset/statefulset.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# -# -# Copyright © 2022 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: external-access -spec: - replicas: 10 - selector: - matchLabels: - app: external-access - template: - metadata: - labels: - app: external-access - spec: - containers: - - name: busybox - image: gcr.io/google_containers/busybox:1.27 - command: ["/bin/sh", "-c", "sleep 3600"] - volumeMounts: - - name: www - mountPath: /data - volumeClaimTemplates: - - metadata: - name: www - annotations: - volume.beta.kubernetes.io/storage-class: external-access-sc - spec: - accessModes: ["ReadWriteMany"] - resources: - requests: - storage: 3Gi diff --git a/tests/e2e/k8s/utils.go b/tests/e2e/k8s/utils.go deleted file mode 100644 index 72732d02..00000000 --- a/tests/e2e/k8s/utils.go +++ /dev/null @@ -1,346 +0,0 @@ -/* - * - * Copyright © 2022-2025 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package e2etest - -import ( - "context" - "fmt" - "os" - "path/filepath" - "time" - - "github.com/onsi/gomega" - "gopkg.in/yaml.v2" - apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" - "k8s.io/kubernetes/test/e2e/framework/deployment" - "k8s.io/kubernetes/test/e2e/framework/manifest" - fpv "k8s.io/kubernetes/test/e2e/framework/pv" - fss "k8s.io/kubernetes/test/e2e/framework/statefulset" - - "k8s.io/kubernetes/test/e2e/framework/testfiles" - - "github.com/onsi/ginkgo/v2" -) - -// getNamespaceToRunTests returns the namespace in which the tests are expected -// to run. For test setups, returns random namespace name -func getNamespaceToRunTests(f *framework.Framework) string { - return f.Namespace.Name -} - -// not usedgit git -// bootstrap function takes care of initializing necessary tests context for e2e tests -func bootstrap(withoutDc ...bool) { - var err error - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // ctx - _, cancel := context.WithCancel(context.Background()) - defer cancel() - - if framework.TestContext.RepoRoot != "" { - testfiles.AddFileSource(testfiles.RootFileSource{Root: framework.TestContext.RepoRoot}) - } - framework.TestContext.Provider = "local" -} - -// getPvFromClaim returns PersistentVolume for requested claim. -func getPvFromClaim(client clientset.Interface, namespace string, claimName string) *v1.PersistentVolume { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - pvclaim, err := client.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, claimName, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pv, err := client.CoreV1().PersistentVolumes().Get(ctx, pvclaim.Spec.VolumeName, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return pv -} - -// DeleteDeployment : delete on the basis of namespace and deployment name -func DeleteDeployment(client clientset.Interface, deploymentObject *apps.Deployment, namespace string) { - deletePolicy := metav1.DeletePropagationForeground - deploymentsClient := client.AppsV1().Deployments(namespace) - err := deploymentsClient.Delete(context.TODO(), deploymentObject.Name, metav1.DeleteOptions{ - PropagationPolicy: &deletePolicy, - }) - framework.ExpectNoError(err) -} - -// ScaleDownDeployment : Decreasing the replica count -func ScaleDownDeployment(client clientset.Interface, deploymentObject *apps.Deployment, namespace string, replicaCount int32) { - s, err := client.AppsV1(). - Deployments(namespace). - GetScale(context.TODO(), deploymentObject.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - sc := *s - sc.Spec.Replicas = replicaCount - - _, err = client.AppsV1(). - Deployments(namespace). - UpdateScale(context.TODO(), - deploymentObject.Name, &sc, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // wait for few seconds to delete pods - for i := 0; i < 2; i++ { - time.Sleep(10 * time.Second) - s, err := client.AppsV1(). - Deployments(namespace). - GetScale(context.TODO(), deploymentObject.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - if s.Spec.Replicas == replicaCount { - break - } - } -} - -// CreateStatefulSet creates a StatefulSet from the manifest at manifestPath in the given namespace. -func CreateStatefulSet(ns string, ss *apps.StatefulSet, c clientset.Interface) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - framework.Logf(fmt.Sprintf("Creating statefulset %v/%v with %d replicas and selector %+v", - ss.Namespace, ss.Name, *(ss.Spec.Replicas), ss.Spec.Selector)) - - _, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{}) - framework.ExpectNoError(err) - fss.WaitForRunningAndReady(ctx, c, *ss.Spec.Replicas, ss) -} - -// CreateDeployment creates a deployment in the provided namespace -func CreateDeployment() (ns string, ss *apps.Deployment, c clientset.Interface) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - framework.Logf(fmt.Sprintf("Creating Deployment %v/%v with %d replicas and selector %+v", - ss.Namespace, ss.Name, *(ss.Spec.Replicas), ss.Spec.Selector)) - - _, err := c.AppsV1().Deployments(ns).Create(ctx, ss, metav1.CreateOptions{}) - framework.ExpectNoError(err) - err = deployment.WaitForDeploymentComplete(c, ss) - framework.ExpectNoError(err) - return -} - -// GetStatefulSetFromManifest creates a StatefulSet from the statefulset.yaml -// file present in the manifest path. -func GetStatefulSetFromManifest(ns string) *apps.StatefulSet { - pwd, _ := os.Getwd() - manifestPath := pwd + "/testing-manifests/statefulset/" - - ssManifestFilePath := filepath.Join(manifestPath, "statefulset.yaml") - framework.Logf("Parsing statefulset from %v", ssManifestFilePath) - ss, err := manifest.StatefulSetFromManifest(ssManifestFilePath, ns) - framework.ExpectNoError(err) - return ss -} - -// createPVCAndStorageClass helps creates a storage class with specified name, -// storageclass parameters and PVC using storage class. -func createPVCAndStorageClass(client clientset.Interface, pvcnamespace string, - pvclaimlabels map[string]string, testParameters map[interface{}]interface{}, ds string, - allowedTopologies []v1.TopologySelectorLabelRequirement, bindingMode storagev1.VolumeBindingMode, - allowVolumeExpansion bool, accessMode v1.PersistentVolumeAccessMode, - names ...string, -) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, error) { - scName := "" - if len(names) > 0 { - scName = names[0] - } - - storageclass, err := createStorageClass(client, testParameters, - allowedTopologies, "", bindingMode, allowVolumeExpansion, scName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - pvclaim, err := createPVC(client, pvcnamespace, pvclaimlabels, ds, storageclass, accessMode) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - return storageclass, pvclaim, err -} - -// createStorageClass helps creates a storage class with specified name, -// storageclass parameters. -func createStorageClass(client clientset.Interface, testParameters map[interface{}]interface{}, - allowedTopologies []v1.TopologySelectorLabelRequirement, - scReclaimPolicy v1.PersistentVolumeReclaimPolicy, bindingMode storagev1.VolumeBindingMode, - allowVolumeExpansion bool, scName string, -) (*storagev1.StorageClass, error) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var storageclass *storagev1.StorageClass - var err error - isStorageClassPresent := false - // since array's credentials are present in testParameters map, setting it as nil to not print sensitive data - testParameters["externalAccess"] = nil - ginkgo.By(fmt.Sprintf("Creating StorageClass %s with scParameters: %+v and allowedTopologies: %+v "+ - "and ReclaimPolicy: %+v and allowVolumeExpansion: %t", - scName, testParameters, allowedTopologies, scReclaimPolicy, allowVolumeExpansion)) - - storageclass, err = client.StorageV1().StorageClasses().Get(ctx, scName, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - gomega.Expect(err).To(gomega.HaveOccurred()) - } - - if storageclass != nil && err == nil { - isStorageClassPresent = true - } - - if !isStorageClassPresent { - storageclass, err = client.StorageV1().StorageClasses().Create(ctx, getStorageClassSpec(scName, - testParameters, allowedTopologies, scReclaimPolicy, bindingMode, allowVolumeExpansion), metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Failed to create storage class with err: %v", err)) - } - - return storageclass, err -} - -// getStorageClassSpec returns Storage Class Spec with supplied storage -// class parameters. -func getStorageClassSpec(scName string, testParameters map[interface{}]interface{}, - allowedTopologies []v1.TopologySelectorLabelRequirement, scReclaimPolicy v1.PersistentVolumeReclaimPolicy, - bindingMode storagev1.VolumeBindingMode, allowVolumeExpansion bool, -) *storagev1.StorageClass { - /* vals := make([]string, 0) - vals = append(vals, testParameters["e2eCSIDriverName"]) - - topo := v1.TopologySelectorLabelRequirement{ - Key: testParameters["e2eCSIDriverName"] + "/" + testParameters["scParamStorageSystemValue"], - Values: vals, - } - */ - // allowedTopologies = append(allowedTopologies, topo) - - if bindingMode == "" { - bindingMode = storagev1.VolumeBindingWaitForFirstConsumer - } - - sc := &storagev1.StorageClass{ - TypeMeta: metav1.TypeMeta{ - Kind: "StorageClass", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "sc-", - }, - Provisioner: fmt.Sprintf("%v", testParameters["e2eCSIDriverName"]), - VolumeBindingMode: &bindingMode, - AllowVolumeExpansion: &allowVolumeExpansion, - } - // If scName is specified, use that name, else auto-generate storage class - // name. - - if scName != "" { - sc.ObjectMeta = metav1.ObjectMeta{ - Name: scName, - } - } - - if testParameters != nil { - testParametersMap := make(map[string]string) - for k, v := range testParameters { - xType := fmt.Sprintf("%T", v) - // since we are using a common file(e2e-values) for configuration so the easiest way here is to pick the values that is of string not object - if xType == "string" { - testParametersMap[fmt.Sprintf("%v", k)] = fmt.Sprintf("%v", v) - } - } - sc.Parameters = testParametersMap - } - if allowedTopologies != nil { - sc.AllowedTopologies = []v1.TopologySelectorTerm{ - { - MatchLabelExpressions: allowedTopologies, - }, - } - } - if scReclaimPolicy != "" { - sc.ReclaimPolicy = &scReclaimPolicy - } - - return sc -} - -// createPVC helps creates pvc with given namespace and labels using given -// storage class. -func createPVC(client clientset.Interface, pvcnamespace string, pvclaimlabels map[string]string, ds string, - storageclass *storagev1.StorageClass, accessMode v1.PersistentVolumeAccessMode, -) (*v1.PersistentVolumeClaim, error) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - pvcspec := getPersistentVolumeClaimSpecWithStorageClass(pvcnamespace, ds, storageclass, pvclaimlabels, accessMode) - - ginkgo.By(fmt.Sprintf("Creating PVC using the Storage Class %s with disk size %s and labels: %+v accessMode: %+v", - storageclass.Name, ds, pvclaimlabels, accessMode)) - - pvclaim, err := fpv.CreatePVC(ctx, client, pvcnamespace, pvcspec) - - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Failed to create pvc with err: %v", err)) - framework.Logf("PVC created: %v in namespace: %v", pvclaim.Name, pvcnamespace) - return pvclaim, err -} - -// getPersistentVolumeClaimSpecWithStorageClass return the PersistentVolumeClaim -// spec with specified storage class. -func getPersistentVolumeClaimSpecWithStorageClass(namespace string, ds string, storageclass *storagev1.StorageClass, - pvclaimlabels map[string]string, accessMode v1.PersistentVolumeAccessMode, -) *v1.PersistentVolumeClaim { - disksize := fmt.Sprintf("%v", testParameters["diskSize"]) - if ds != "" { - disksize = ds - } - if accessMode == "" { - // If accessMode is not specified, set the default accessMode. - accessMode = v1.ReadWriteOnce - } - claim := &v1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "pvc-", - Namespace: namespace, - }, - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{ - accessMode, - }, - Resources: v1.VolumeResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceName(v1.ResourceStorage): resource.MustParse(disksize), - }, - }, - StorageClassName: &(storageclass.Name), - }, - } - - if pvclaimlabels != nil { - claim.Labels = pvclaimlabels - } - return claim -} - -func readYaml(values string) (map[interface{}]interface{}, error) { - yfile, err := os.ReadFile(filepath.Clean(values)) - if err != nil { - return nil, err - } - data := make(map[interface{}]interface{}) - err = yaml.Unmarshal(yfile, &data) - if err != nil { - return nil, err - } - return data, nil -} diff --git a/tests/sanity/README.md b/tests/sanity/README.md index 226834d6..74cdd7ac 100644 --- a/tests/sanity/README.md +++ b/tests/sanity/README.md @@ -22,7 +22,7 @@ cd csi-powerstore/ make build ``` -3. Fill in the following files in tests/sanity/; anything with a "REPLACE" prefix needs to be replaced with a real value: +3. Fill in the following files in tests/sanity/; anything with a "REPLACE" prefix needs to be replaced with a real value: - config.yaml, this file will be used by the binary built in step 2 (from now on, referred to as "the binary" for short) to connect to array - setup-driver-controller-sanity.sh, this file is used to start the driver's controller service from the binary @@ -30,6 +30,11 @@ make build - params.yaml, this file is used by the sanity test to pass in parameters that would be defined in the storageclass - [Optional] driver-config-params.yaml, this file controls how the binary's logger is configured +4. Install Disaster Recovery (DR) CRD from operatorconfig/moduleconfig/common/disaster-recovery/dr-crds.yaml in csm-operator repository. +```sh +kubectl apply -f dr-crds.yaml +``` + ## Running 1. Run the shell script to setup the driver's node service @@ -53,7 +58,7 @@ make build 3. In (another) new terminal window, run the shell script to run the sanity test ```sh -./run-csi-sanity.sh +./run-csi-sanity.sh ``` Tests should pass in 10-12 minutes diff --git a/tests/sanity/setup-driver-controller-sanity.sh b/tests/sanity/setup-driver-controller-sanity.sh index f7e93d57..eb713c20 100755 --- a/tests/sanity/setup-driver-controller-sanity.sh +++ b/tests/sanity/setup-driver-controller-sanity.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright © 2020-2025 Dell Inc. or its subsidiaries. All Rights Reserved. +# Copyright © 2020-2026 Dell Inc. or its subsidiaries. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ export CSI_ENDPOINT=$(pwd)/controller.sock export X_CSI_VOL_PREFIX=sanity export X_CSI_MAX_VOLUMES_PER_NODE=0 export X_CSI_NODE_IP=REPLACE_IP -export X_CSI_NODE_NAME=REPLACE_HOSTNAME +export X_CSI_POWERSTORE_KUBE_NODE_NAME=REPLACE_NAME export X_CSI_HEALTH_MONITOR_ENABLED=true export CSI_AUTO_ROUND_OFF_FILESYSTEM_SIZE=true diff --git a/tests/sanity/setup-driver-node-sanity.sh b/tests/sanity/setup-driver-node-sanity.sh index f11756b1..a7889d1a 100755 --- a/tests/sanity/setup-driver-node-sanity.sh +++ b/tests/sanity/setup-driver-node-sanity.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright © 2020-2025 Dell Inc. or its subsidiaries. All Rights Reserved. +# Copyright © 2020-2026 Dell Inc. or its subsidiaries. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ export CSI_ENDPOINT=$(pwd)/node.sock export X_CSI_VOL_PREFIX=node export X_CSI_MAX_VOLUMES_PER_NODE=0 export X_CSI_NODE_IP=REPLACE_IP -export X_CSI_NODE_NAME=REPLACE_HOSTNAME +export X_CSI_POWERSTORE_KUBE_NODE_NAME=REPLACE_NAME export X_CSI_POWERSTORE_NODE_ID_PATH=/etc/machine-id export X_CSI_POWERSTORE_NODE_CHROOT_PATH=/ export X_CSI_HEALTH_MONITOR_ENABLED=true